2024-11-09 02:21:08,631 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-09 02:21:08,643 main DEBUG Took 0.010351 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-09 02:21:08,643 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-09 02:21:08,644 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-09 02:21:08,645 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-09 02:21:08,646 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:21:08,655 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-09 02:21:08,671 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:21:08,673 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:21:08,674 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:21:08,674 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:21:08,675 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:21:08,675 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:21:08,676 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:21:08,677 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:21:08,677 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:21:08,678 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:21:08,679 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:21:08,679 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:21:08,680 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:21:08,680 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:21:08,681 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:21:08,681 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:21:08,682 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:21:08,682 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:21:08,683 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:21:08,683 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:21:08,684 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:21:08,684 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:21:08,685 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:21:08,685 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:21:08,686 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:21:08,686 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-09 02:21:08,688 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:21:08,690 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-09 02:21:08,692 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-09 02:21:08,692 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-09 02:21:08,694 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-09 02:21:08,695 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-09 02:21:08,704 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-09 02:21:08,707 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-09 02:21:08,710 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-09 02:21:08,710 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-09 02:21:08,711 main DEBUG createAppenders(={Console}) 2024-11-09 02:21:08,712 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-09 02:21:08,712 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-09 02:21:08,712 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-09 02:21:08,713 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-09 02:21:08,713 main DEBUG OutputStream closed 2024-11-09 02:21:08,714 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-09 02:21:08,714 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-09 02:21:08,714 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-09 02:21:08,806 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-09 02:21:08,810 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-09 02:21:08,811 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-09 02:21:08,812 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-09 02:21:08,813 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-09 02:21:08,814 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-09 02:21:08,814 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-09 02:21:08,815 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-09 02:21:08,815 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-09 02:21:08,816 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-09 02:21:08,816 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-09 02:21:08,817 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-09 02:21:08,817 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-09 02:21:08,818 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-09 02:21:08,818 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-09 02:21:08,819 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-09 02:21:08,819 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-09 02:21:08,820 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-09 02:21:08,823 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09 02:21:08,824 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-09 02:21:08,824 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-09 02:21:08,825 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-09T02:21:09,127 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d 2024-11-09 02:21:09,131 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-09 02:21:09,131 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09T02:21:09,142 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-09T02:21:09,171 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-09T02:21:09,188 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/cluster_76eb7638-e5d7-2221-3a01-3d32335e942f, deleteOnExit=true 2024-11-09T02:21:09,189 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-09T02:21:09,190 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/test.cache.data in system properties and HBase conf 2024-11-09T02:21:09,191 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/hadoop.tmp.dir in system properties and HBase conf 2024-11-09T02:21:09,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/hadoop.log.dir in system properties and HBase conf 2024-11-09T02:21:09,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-09T02:21:09,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-09T02:21:09,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-09T02:21:09,304 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-09T02:21:09,446 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T02:21:09,458 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-09T02:21:09,459 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-09T02:21:09,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-09T02:21:09,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T02:21:09,464 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-09T02:21:09,466 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-09T02:21:09,468 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T02:21:09,469 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T02:21:09,469 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-09T02:21:09,470 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/nfs.dump.dir in system properties and HBase conf 2024-11-09T02:21:09,470 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/java.io.tmpdir in system properties and HBase conf 2024-11-09T02:21:09,471 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T02:21:09,471 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-09T02:21:09,472 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-09T02:21:10,803 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-09T02:21:10,902 INFO [Time-limited test {}] log.Log(170): Logging initialized @3052ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-09T02:21:10,973 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T02:21:11,043 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T02:21:11,071 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T02:21:11,071 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T02:21:11,073 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-09T02:21:11,089 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T02:21:11,092 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/hadoop.log.dir/,AVAILABLE} 2024-11-09T02:21:11,093 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T02:21:11,303 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/java.io.tmpdir/jetty-localhost-40323-hadoop-hdfs-3_4_1-tests_jar-_-any-2885852135175459957/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T02:21:11,310 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:40323} 2024-11-09T02:21:11,310 INFO [Time-limited test {}] server.Server(415): Started @3461ms 2024-11-09T02:21:11,849 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T02:21:11,856 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T02:21:11,859 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T02:21:11,859 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T02:21:11,859 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-09T02:21:11,860 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/hadoop.log.dir/,AVAILABLE} 2024-11-09T02:21:11,860 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T02:21:11,968 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/java.io.tmpdir/jetty-localhost-36529-hadoop-hdfs-3_4_1-tests_jar-_-any-5959502462881545768/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T02:21:11,969 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:36529} 2024-11-09T02:21:11,969 INFO [Time-limited test {}] server.Server(415): Started @4120ms 2024-11-09T02:21:12,019 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-09T02:21:13,369 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/cluster_76eb7638-e5d7-2221-3a01-3d32335e942f/data/data2/current/BP-1729710872-172.17.0.3-1731118870108/current, will proceed with Du for space computation calculation, 2024-11-09T02:21:13,369 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/cluster_76eb7638-e5d7-2221-3a01-3d32335e942f/data/data1/current/BP-1729710872-172.17.0.3-1731118870108/current, will proceed with Du for space computation calculation, 2024-11-09T02:21:13,405 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-09T02:21:13,452 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x115c03e15f738e08 with lease ID 0xf43b85fa0b2e50b4: Processing first storage report for DS-63a05a07-d763-4897-8470-d1265edf73c5 from datanode DatanodeRegistration(127.0.0.1:46141, datanodeUuid=e9eef127-1316-4ba3-b291-73c6922bbd01, infoPort=40821, infoSecurePort=0, ipcPort=37657, storageInfo=lv=-57;cid=testClusterID;nsid=1825582511;c=1731118870108) 2024-11-09T02:21:13,453 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x115c03e15f738e08 with lease ID 0xf43b85fa0b2e50b4: from storage DS-63a05a07-d763-4897-8470-d1265edf73c5 node DatanodeRegistration(127.0.0.1:46141, datanodeUuid=e9eef127-1316-4ba3-b291-73c6922bbd01, infoPort=40821, infoSecurePort=0, ipcPort=37657, storageInfo=lv=-57;cid=testClusterID;nsid=1825582511;c=1731118870108), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-09T02:21:13,454 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x115c03e15f738e08 with lease ID 0xf43b85fa0b2e50b4: Processing first storage report for DS-ede2e3f0-b8bf-4bc7-ac3b-1afa33ee3abb from datanode DatanodeRegistration(127.0.0.1:46141, datanodeUuid=e9eef127-1316-4ba3-b291-73c6922bbd01, infoPort=40821, infoSecurePort=0, ipcPort=37657, storageInfo=lv=-57;cid=testClusterID;nsid=1825582511;c=1731118870108) 2024-11-09T02:21:13,454 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x115c03e15f738e08 with lease ID 0xf43b85fa0b2e50b4: from storage DS-ede2e3f0-b8bf-4bc7-ac3b-1afa33ee3abb node DatanodeRegistration(127.0.0.1:46141, datanodeUuid=e9eef127-1316-4ba3-b291-73c6922bbd01, infoPort=40821, infoSecurePort=0, ipcPort=37657, storageInfo=lv=-57;cid=testClusterID;nsid=1825582511;c=1731118870108), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-09T02:21:13,502 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d 2024-11-09T02:21:13,576 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/cluster_76eb7638-e5d7-2221-3a01-3d32335e942f/zookeeper_0, clientPort=62339, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/cluster_76eb7638-e5d7-2221-3a01-3d32335e942f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/cluster_76eb7638-e5d7-2221-3a01-3d32335e942f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-09T02:21:13,587 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62339 2024-11-09T02:21:13,600 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T02:21:13,603 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T02:21:13,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741825_1001 (size=7) 2024-11-09T02:21:14,252 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a with version=8 2024-11-09T02:21:14,253 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/hbase-staging 2024-11-09T02:21:14,349 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-09T02:21:14,631 INFO [Time-limited test {}] client.ConnectionUtils(128): master/53f286d3af3b:0 server-side Connection retries=45 2024-11-09T02:21:14,641 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T02:21:14,642 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T02:21:14,647 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T02:21:14,647 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T02:21:14,648 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T02:21:14,793 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-09T02:21:14,851 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-09T02:21:14,859 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-09T02:21:14,863 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T02:21:14,886 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 30016 (auto-detected) 2024-11-09T02:21:14,887 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-09T02:21:14,905 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:45041 2024-11-09T02:21:14,926 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45041 connecting to ZooKeeper ensemble=127.0.0.1:62339 2024-11-09T02:21:15,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:450410x0, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T02:21:15,010 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45041-0x1011d65b7800000 connected 2024-11-09T02:21:15,098 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T02:21:15,101 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T02:21:15,110 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T02:21:15,113 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a, hbase.cluster.distributed=false 2024-11-09T02:21:15,136 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T02:21:15,144 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45041 2024-11-09T02:21:15,145 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45041 2024-11-09T02:21:15,146 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45041 2024-11-09T02:21:15,147 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45041 2024-11-09T02:21:15,148 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45041 2024-11-09T02:21:15,254 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/53f286d3af3b:0 server-side Connection retries=45 2024-11-09T02:21:15,255 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T02:21:15,256 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T02:21:15,256 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T02:21:15,256 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T02:21:15,256 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T02:21:15,258 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T02:21:15,261 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T02:21:15,262 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46079 2024-11-09T02:21:15,264 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46079 connecting to ZooKeeper ensemble=127.0.0.1:62339 2024-11-09T02:21:15,265 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T02:21:15,267 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T02:21:15,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:460790x0, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T02:21:15,281 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:460790x0, quorum=127.0.0.1:62339, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T02:21:15,281 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46079-0x1011d65b7800001 connected 2024-11-09T02:21:15,285 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-09T02:21:15,293 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-09T02:21:15,296 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46079-0x1011d65b7800001, quorum=127.0.0.1:62339, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T02:21:15,301 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46079-0x1011d65b7800001, quorum=127.0.0.1:62339, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T02:21:15,302 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46079 2024-11-09T02:21:15,303 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46079 2024-11-09T02:21:15,303 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46079 2024-11-09T02:21:15,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46079 2024-11-09T02:21:15,305 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46079 2024-11-09T02:21:15,320 DEBUG [M:0;53f286d3af3b:45041 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;53f286d3af3b:45041 2024-11-09T02:21:15,321 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/53f286d3af3b,45041,1731118874457 2024-11-09T02:21:15,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T02:21:15,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46079-0x1011d65b7800001, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T02:21:15,340 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/53f286d3af3b,45041,1731118874457 2024-11-09T02:21:15,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:21:15,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46079-0x1011d65b7800001, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T02:21:15,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46079-0x1011d65b7800001, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:21:15,381 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-09T02:21:15,382 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/53f286d3af3b,45041,1731118874457 from backup master directory 2024-11-09T02:21:15,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/53f286d3af3b,45041,1731118874457 2024-11-09T02:21:15,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46079-0x1011d65b7800001, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T02:21:15,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T02:21:15,397 WARN [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T02:21:15,397 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=53f286d3af3b,45041,1731118874457 2024-11-09T02:21:15,399 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-09T02:21:15,401 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-09T02:21:15,455 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/hbase.id] with ID: 8ede6ca3-e8ea-4612-bac6-a05540447f39 2024-11-09T02:21:15,455 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/.tmp/hbase.id 2024-11-09T02:21:15,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741826_1002 (size=42) 2024-11-09T02:21:15,866 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/.tmp/hbase.id]:[hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/hbase.id] 2024-11-09T02:21:15,908 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T02:21:15,913 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-09T02:21:15,933 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-09T02:21:16,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46079-0x1011d65b7800001, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:21:16,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:21:16,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741827_1003 (size=196) 2024-11-09T02:21:16,502 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:21:16,505 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-09T02:21:16,526 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:16,531 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T02:21:16,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741828_1004 (size=1189) 2024-11-09T02:21:16,980 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/MasterData/data/master/store 2024-11-09T02:21:16,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741829_1005 (size=34) 2024-11-09T02:21:17,403 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-09T02:21:17,406 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:21:17,407 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T02:21:17,407 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T02:21:17,407 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T02:21:17,409 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T02:21:17,409 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T02:21:17,409 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T02:21:17,410 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731118877407Disabling compacts and flushes for region at 1731118877407Disabling writes for close at 1731118877409 (+2 ms)Writing region close event to WAL at 1731118877409Closed at 1731118877409 2024-11-09T02:21:17,413 WARN [master/53f286d3af3b:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/MasterData/data/master/store/.initializing 2024-11-09T02:21:17,413 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/MasterData/WALs/53f286d3af3b,45041,1731118874457 2024-11-09T02:21:17,423 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T02:21:17,436 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=53f286d3af3b%2C45041%2C1731118874457, suffix=, logDir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/MasterData/WALs/53f286d3af3b,45041,1731118874457, archiveDir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/MasterData/oldWALs, maxLogs=10 2024-11-09T02:21:17,459 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/MasterData/WALs/53f286d3af3b,45041,1731118874457/53f286d3af3b%2C45041%2C1731118874457.1731118877440, exclude list is [], retry=0 2024-11-09T02:21:17,486 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46141,DS-63a05a07-d763-4897-8470-d1265edf73c5,DISK] 2024-11-09T02:21:17,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-09T02:21:17,524 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/MasterData/WALs/53f286d3af3b,45041,1731118874457/53f286d3af3b%2C45041%2C1731118874457.1731118877440 2024-11-09T02:21:17,524 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40821:40821)] 2024-11-09T02:21:17,525 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:21:17,525 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:21:17,528 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:21:17,529 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:21:17,565 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:21:17,593 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-09T02:21:17,597 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:17,600 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:21:17,600 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:21:17,605 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-09T02:21:17,605 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:17,607 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:21:17,607 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:21:17,610 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-09T02:21:17,610 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:17,611 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:21:17,612 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:21:17,615 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-09T02:21:17,615 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:17,616 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:21:17,617 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:21:17,620 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:21:17,621 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:21:17,627 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:21:17,628 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:21:17,632 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T02:21:17,636 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:21:17,641 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:21:17,642 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69391002, jitterRate=0.0340065062046051}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T02:21:17,652 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731118877541Initializing all the Stores at 1731118877543 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731118877544 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118877544Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118877545 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118877545Cleaning up temporary data from old regions at 1731118877628 (+83 ms)Region opened successfully at 1731118877651 (+23 ms) 2024-11-09T02:21:17,653 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-09T02:21:17,693 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30e1b230, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=53f286d3af3b/172.17.0.3:0 2024-11-09T02:21:17,731 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-09T02:21:17,745 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-09T02:21:17,745 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-09T02:21:17,749 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T02:21:17,750 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-09T02:21:17,756 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-09T02:21:17,756 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-09T02:21:17,788 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-09T02:21:17,799 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-09T02:21:17,811 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-09T02:21:17,814 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-09T02:21:17,816 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-09T02:21:17,827 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-09T02:21:17,830 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-09T02:21:17,834 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-09T02:21:17,917 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-09T02:21:17,918 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-09T02:21:17,927 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-09T02:21:17,941 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-09T02:21:17,953 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-09T02:21:17,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T02:21:17,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46079-0x1011d65b7800001, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T02:21:17,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46079-0x1011d65b7800001, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:21:17,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:21:17,967 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=53f286d3af3b,45041,1731118874457, sessionid=0x1011d65b7800000, setting cluster-up flag (Was=false) 2024-11-09T02:21:17,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:21:17,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46079-0x1011d65b7800001, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:21:18,027 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-09T02:21:18,029 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=53f286d3af3b,45041,1731118874457 2024-11-09T02:21:18,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46079-0x1011d65b7800001, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:21:18,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:21:18,085 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-09T02:21:18,086 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=53f286d3af3b,45041,1731118874457 2024-11-09T02:21:18,092 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-09T02:21:18,112 INFO [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer(746): ClusterId : 8ede6ca3-e8ea-4612-bac6-a05540447f39 2024-11-09T02:21:18,114 DEBUG [RS:0;53f286d3af3b:46079 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-09T02:21:18,129 DEBUG [RS:0;53f286d3af3b:46079 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-09T02:21:18,129 DEBUG [RS:0;53f286d3af3b:46079 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-09T02:21:18,139 DEBUG [RS:0;53f286d3af3b:46079 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-09T02:21:18,139 DEBUG [RS:0;53f286d3af3b:46079 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a305926, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=53f286d3af3b/172.17.0.3:0 2024-11-09T02:21:18,158 DEBUG [RS:0;53f286d3af3b:46079 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;53f286d3af3b:46079 2024-11-09T02:21:18,161 INFO [RS:0;53f286d3af3b:46079 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-09T02:21:18,161 INFO [RS:0;53f286d3af3b:46079 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-09T02:21:18,161 DEBUG [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-09T02:21:18,164 INFO [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer(2659): reportForDuty to master=53f286d3af3b,45041,1731118874457 with port=46079, startcode=1731118875223 2024-11-09T02:21:18,167 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T02:21:18,173 DEBUG [RS:0;53f286d3af3b:46079 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T02:21:18,176 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-09T02:21:18,183 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-09T02:21:18,189 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 53f286d3af3b,45041,1731118874457 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-09T02:21:18,197 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/53f286d3af3b:0, corePoolSize=5, maxPoolSize=5 2024-11-09T02:21:18,198 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/53f286d3af3b:0, corePoolSize=5, maxPoolSize=5 2024-11-09T02:21:18,198 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/53f286d3af3b:0, corePoolSize=5, maxPoolSize=5 2024-11-09T02:21:18,198 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/53f286d3af3b:0, corePoolSize=5, maxPoolSize=5 2024-11-09T02:21:18,198 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/53f286d3af3b:0, corePoolSize=10, maxPoolSize=10 2024-11-09T02:21:18,199 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:21:18,199 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/53f286d3af3b:0, corePoolSize=2, maxPoolSize=2 2024-11-09T02:21:18,199 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:21:18,205 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118908204 2024-11-09T02:21:18,206 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-09T02:21:18,207 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T02:21:18,208 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-09T02:21:18,208 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-09T02:21:18,208 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60309, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T02:21:18,211 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-09T02:21:18,212 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-09T02:21:18,212 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-09T02:21:18,212 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-09T02:21:18,213 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:18,213 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-09T02:21:18,213 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T02:21:18,216 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45041 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:18,218 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-09T02:21:18,219 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-09T02:21:18,220 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-09T02:21:18,224 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-09T02:21:18,225 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-09T02:21:18,227 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/53f286d3af3b:0:becomeActiveMaster-HFileCleaner.large.0-1731118878226,5,FailOnTimeoutGroup] 2024-11-09T02:21:18,228 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/53f286d3af3b:0:becomeActiveMaster-HFileCleaner.small.0-1731118878227,5,FailOnTimeoutGroup] 2024-11-09T02:21:18,228 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T02:21:18,228 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-09T02:21:18,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741831_1007 (size=1321) 2024-11-09T02:21:18,230 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-09T02:21:18,230 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-09T02:21:18,233 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-09T02:21:18,234 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a 2024-11-09T02:21:18,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741832_1008 (size=32) 2024-11-09T02:21:18,247 DEBUG [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-09T02:21:18,247 WARN [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-09T02:21:18,348 INFO [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer(2659): reportForDuty to master=53f286d3af3b,45041,1731118874457 with port=46079, startcode=1731118875223 2024-11-09T02:21:18,350 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45041 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 53f286d3af3b,46079,1731118875223 2024-11-09T02:21:18,352 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45041 {}] master.ServerManager(517): Registering regionserver=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:18,360 DEBUG [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a 2024-11-09T02:21:18,360 DEBUG [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45877 2024-11-09T02:21:18,360 DEBUG [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-09T02:21:18,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T02:21:18,375 DEBUG [RS:0;53f286d3af3b:46079 {}] zookeeper.ZKUtil(111): regionserver:46079-0x1011d65b7800001, quorum=127.0.0.1:62339, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/53f286d3af3b,46079,1731118875223 2024-11-09T02:21:18,375 WARN [RS:0;53f286d3af3b:46079 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T02:21:18,375 INFO [RS:0;53f286d3af3b:46079 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T02:21:18,376 DEBUG [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/WALs/53f286d3af3b,46079,1731118875223 2024-11-09T02:21:18,377 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [53f286d3af3b,46079,1731118875223] 2024-11-09T02:21:18,401 INFO [RS:0;53f286d3af3b:46079 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-09T02:21:18,416 INFO [RS:0;53f286d3af3b:46079 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-09T02:21:18,422 INFO [RS:0;53f286d3af3b:46079 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-09T02:21:18,422 INFO [RS:0;53f286d3af3b:46079 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T02:21:18,423 INFO [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-09T02:21:18,428 INFO [RS:0;53f286d3af3b:46079 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-09T02:21:18,430 INFO [RS:0;53f286d3af3b:46079 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-09T02:21:18,430 DEBUG [RS:0;53f286d3af3b:46079 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:21:18,430 DEBUG [RS:0;53f286d3af3b:46079 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:21:18,430 DEBUG [RS:0;53f286d3af3b:46079 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:21:18,430 DEBUG [RS:0;53f286d3af3b:46079 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:21:18,431 DEBUG [RS:0;53f286d3af3b:46079 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:21:18,431 DEBUG [RS:0;53f286d3af3b:46079 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0, corePoolSize=2, maxPoolSize=2 2024-11-09T02:21:18,431 DEBUG [RS:0;53f286d3af3b:46079 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:21:18,431 DEBUG [RS:0;53f286d3af3b:46079 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:21:18,432 DEBUG [RS:0;53f286d3af3b:46079 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:21:18,432 DEBUG [RS:0;53f286d3af3b:46079 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:21:18,432 DEBUG [RS:0;53f286d3af3b:46079 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:21:18,432 DEBUG [RS:0;53f286d3af3b:46079 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:21:18,432 DEBUG [RS:0;53f286d3af3b:46079 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/53f286d3af3b:0, corePoolSize=3, maxPoolSize=3 2024-11-09T02:21:18,433 DEBUG [RS:0;53f286d3af3b:46079 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0, corePoolSize=3, maxPoolSize=3 2024-11-09T02:21:18,433 INFO [RS:0;53f286d3af3b:46079 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T02:21:18,434 INFO [RS:0;53f286d3af3b:46079 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T02:21:18,434 INFO [RS:0;53f286d3af3b:46079 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T02:21:18,434 INFO [RS:0;53f286d3af3b:46079 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-09T02:21:18,434 INFO [RS:0;53f286d3af3b:46079 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-09T02:21:18,434 INFO [RS:0;53f286d3af3b:46079 {}] hbase.ChoreService(168): Chore ScheduledChore name=53f286d3af3b,46079,1731118875223-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T02:21:18,454 INFO [RS:0;53f286d3af3b:46079 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-09T02:21:18,455 INFO [RS:0;53f286d3af3b:46079 {}] hbase.ChoreService(168): Chore ScheduledChore name=53f286d3af3b,46079,1731118875223-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T02:21:18,455 INFO [RS:0;53f286d3af3b:46079 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T02:21:18,456 INFO [RS:0;53f286d3af3b:46079 {}] regionserver.Replication(171): 53f286d3af3b,46079,1731118875223 started 2024-11-09T02:21:18,472 INFO [RS:0;53f286d3af3b:46079 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T02:21:18,472 INFO [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer(1482): Serving as 53f286d3af3b,46079,1731118875223, RpcServer on 53f286d3af3b/172.17.0.3:46079, sessionid=0x1011d65b7800001 2024-11-09T02:21:18,473 DEBUG [RS:0;53f286d3af3b:46079 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-09T02:21:18,473 DEBUG [RS:0;53f286d3af3b:46079 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 53f286d3af3b,46079,1731118875223 2024-11-09T02:21:18,473 DEBUG [RS:0;53f286d3af3b:46079 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '53f286d3af3b,46079,1731118875223' 2024-11-09T02:21:18,473 DEBUG [RS:0;53f286d3af3b:46079 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-09T02:21:18,474 DEBUG [RS:0;53f286d3af3b:46079 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-09T02:21:18,475 DEBUG [RS:0;53f286d3af3b:46079 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-09T02:21:18,475 DEBUG [RS:0;53f286d3af3b:46079 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-09T02:21:18,475 DEBUG [RS:0;53f286d3af3b:46079 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 53f286d3af3b,46079,1731118875223 2024-11-09T02:21:18,475 DEBUG [RS:0;53f286d3af3b:46079 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '53f286d3af3b,46079,1731118875223' 2024-11-09T02:21:18,475 DEBUG [RS:0;53f286d3af3b:46079 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-09T02:21:18,476 DEBUG [RS:0;53f286d3af3b:46079 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-09T02:21:18,477 DEBUG [RS:0;53f286d3af3b:46079 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-09T02:21:18,477 INFO [RS:0;53f286d3af3b:46079 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-09T02:21:18,477 INFO [RS:0;53f286d3af3b:46079 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-09T02:21:18,582 INFO [RS:0;53f286d3af3b:46079 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T02:21:18,585 INFO [RS:0;53f286d3af3b:46079 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=53f286d3af3b%2C46079%2C1731118875223, suffix=, logDir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/WALs/53f286d3af3b,46079,1731118875223, archiveDir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/oldWALs, maxLogs=32 2024-11-09T02:21:18,604 DEBUG [RS:0;53f286d3af3b:46079 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/WALs/53f286d3af3b,46079,1731118875223/53f286d3af3b%2C46079%2C1731118875223.1731118878587, exclude list is [], retry=0 2024-11-09T02:21:18,608 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46141,DS-63a05a07-d763-4897-8470-d1265edf73c5,DISK] 2024-11-09T02:21:18,611 INFO [RS:0;53f286d3af3b:46079 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/WALs/53f286d3af3b,46079,1731118875223/53f286d3af3b%2C46079%2C1731118875223.1731118878587 2024-11-09T02:21:18,612 DEBUG [RS:0;53f286d3af3b:46079 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40821:40821)] 2024-11-09T02:21:18,648 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:21:18,651 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T02:21:18,653 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T02:21:18,653 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:18,655 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:21:18,655 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-09T02:21:18,657 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T02:21:18,657 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:18,658 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:21:18,659 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T02:21:18,661 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T02:21:18,661 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:18,662 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:21:18,662 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T02:21:18,665 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T02:21:18,665 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:18,666 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:21:18,666 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T02:21:18,668 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/1588230740 2024-11-09T02:21:18,669 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/1588230740 2024-11-09T02:21:18,671 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T02:21:18,671 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T02:21:18,672 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T02:21:18,674 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T02:21:18,678 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:21:18,679 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67608908, jitterRate=0.007451236248016357}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T02:21:18,682 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731118878648Initializing all the Stores at 1731118878650 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731118878650Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731118878650Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118878650Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731118878650Cleaning up temporary data from old regions at 1731118878671 (+21 ms)Region opened successfully at 1731118878682 (+11 ms) 2024-11-09T02:21:18,682 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T02:21:18,682 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T02:21:18,683 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T02:21:18,683 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T02:21:18,683 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T02:21:18,684 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T02:21:18,684 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731118878682Disabling compacts and flushes for region at 1731118878682Disabling writes for close at 1731118878683 (+1 ms)Writing region close event to WAL at 1731118878684 (+1 ms)Closed at 1731118878684 2024-11-09T02:21:18,688 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T02:21:18,688 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-09T02:21:18,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-09T02:21:18,703 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:21:18,706 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-09T02:21:18,859 DEBUG [53f286d3af3b:45041 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-09T02:21:18,871 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:18,877 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,46079,1731118875223, state=OPENING 2024-11-09T02:21:18,927 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-09T02:21:18,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:21:18,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46079-0x1011d65b7800001, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:21:18,938 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:21:18,938 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:21:18,940 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:21:18,941 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,46079,1731118875223}] 2024-11-09T02:21:19,113 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-09T02:21:19,116 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46701, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-09T02:21:19,128 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-09T02:21:19,128 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T02:21:19,129 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-09T02:21:19,132 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=53f286d3af3b%2C46079%2C1731118875223.meta, suffix=.meta, logDir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/WALs/53f286d3af3b,46079,1731118875223, archiveDir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/oldWALs, maxLogs=32 2024-11-09T02:21:19,151 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/WALs/53f286d3af3b,46079,1731118875223/53f286d3af3b%2C46079%2C1731118875223.meta.1731118879135.meta, exclude list is [], retry=0 2024-11-09T02:21:19,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46141,DS-63a05a07-d763-4897-8470-d1265edf73c5,DISK] 2024-11-09T02:21:19,158 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/WALs/53f286d3af3b,46079,1731118875223/53f286d3af3b%2C46079%2C1731118875223.meta.1731118879135.meta 2024-11-09T02:21:19,159 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40821:40821)] 2024-11-09T02:21:19,160 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:21:19,162 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-09T02:21:19,164 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-09T02:21:19,169 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-09T02:21:19,174 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-09T02:21:19,174 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:21:19,175 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-09T02:21:19,175 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-09T02:21:19,178 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T02:21:19,179 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T02:21:19,179 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:19,180 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:21:19,180 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-09T02:21:19,182 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T02:21:19,182 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:19,182 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:21:19,183 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T02:21:19,184 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T02:21:19,184 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:19,185 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:21:19,185 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T02:21:19,186 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T02:21:19,187 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:19,187 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:21:19,188 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T02:21:19,189 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/1588230740 2024-11-09T02:21:19,192 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/1588230740 2024-11-09T02:21:19,195 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T02:21:19,195 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T02:21:19,196 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T02:21:19,199 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T02:21:19,201 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71770945, jitterRate=0.06947042047977448}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T02:21:19,201 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-09T02:21:19,203 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731118879175Writing region info on filesystem at 1731118879175Initializing all the Stores at 1731118879177 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731118879177Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731118879177Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118879178 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731118879178Cleaning up temporary data from old regions at 1731118879195 (+17 ms)Running coprocessor post-open hooks at 1731118879201 (+6 ms)Region opened successfully at 1731118879202 (+1 ms) 2024-11-09T02:21:19,209 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731118879106 2024-11-09T02:21:19,222 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-09T02:21:19,223 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-09T02:21:19,223 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:19,225 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,46079,1731118875223, state=OPEN 2024-11-09T02:21:19,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:21:19,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46079-0x1011d65b7800001, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:21:19,268 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:21:19,268 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:21:19,268 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:19,274 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-09T02:21:19,275 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,46079,1731118875223 in 327 msec 2024-11-09T02:21:19,281 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-09T02:21:19,281 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 582 msec 2024-11-09T02:21:19,283 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T02:21:19,283 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-09T02:21:19,301 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:21:19,302 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:21:19,322 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:19,324 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39451, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:19,344 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2180 sec 2024-11-09T02:21:19,344 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731118879344, completionTime=-1 2024-11-09T02:21:19,347 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-09T02:21:19,347 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T02:21:19,377 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-09T02:21:19,377 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118939377 2024-11-09T02:21:19,377 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118999377 2024-11-09T02:21:19,377 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 30 msec 2024-11-09T02:21:19,380 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=53f286d3af3b,45041,1731118874457-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T02:21:19,380 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=53f286d3af3b,45041,1731118874457-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T02:21:19,380 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=53f286d3af3b,45041,1731118874457-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T02:21:19,382 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-53f286d3af3b:45041, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T02:21:19,382 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-09T02:21:19,383 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-09T02:21:19,414 DEBUG [master/53f286d3af3b:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-09T02:21:19,447 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.050sec 2024-11-09T02:21:19,449 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-09T02:21:19,450 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-09T02:21:19,451 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-09T02:21:19,451 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-09T02:21:19,452 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-09T02:21:19,453 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=53f286d3af3b,45041,1731118874457-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T02:21:19,453 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=53f286d3af3b,45041,1731118874457-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-09T02:21:19,462 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-09T02:21:19,463 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-09T02:21:19,464 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=53f286d3af3b,45041,1731118874457-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T02:21:19,521 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:19,523 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-09T02:21:19,523 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-09T02:21:19,525 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:21:19,527 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:21:19,534 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:21:19,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:21:19,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:21:19,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46a49a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:19,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:21:19,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:21:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:19,543 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:21:19,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:19,546 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:21:19,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:21:19,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:19,555 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:19,573 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=53f286d3af3b,45041,1731118874457 2024-11-09T02:21:19,583 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=474, ProcessCount=11, AvailableMemoryMB=2421 2024-11-09T02:21:19,590 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T02:21:19,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 53f286d3af3b,45041,1731118874457 2024-11-09T02:21:19,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3108930d 2024-11-09T02:21:19,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T02:21:19,601 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T02:21:19,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T02:21:19,609 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:21:19,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T02:21:19,622 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:21:19,624 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-09T02:21:19,625 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:19,627 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:21:19,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T02:21:19,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741835_1011 (size=959) 2024-11-09T02:21:19,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T02:21:19,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T02:21:20,046 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a 2024-11-09T02:21:20,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741836_1012 (size=53) 2024-11-09T02:21:20,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T02:21:20,455 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:21:20,456 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing aad4fc6835c8f9bd4339c4a54a2c56fe, disabling compactions & flushes 2024-11-09T02:21:20,456 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:20,456 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:20,456 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. after waiting 0 ms 2024-11-09T02:21:20,456 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:20,456 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:20,456 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for aad4fc6835c8f9bd4339c4a54a2c56fe: Waiting for close lock at 1731118880456Disabling compacts and flushes for region at 1731118880456Disabling writes for close at 1731118880456Writing region close event to WAL at 1731118880456Closed at 1731118880456 2024-11-09T02:21:20,458 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:21:20,463 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731118880458"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118880458"}]},"ts":"1731118880458"} 2024-11-09T02:21:20,467 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T02:21:20,469 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:21:20,471 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118880469"}]},"ts":"1731118880469"} 2024-11-09T02:21:20,476 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T02:21:20,477 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aad4fc6835c8f9bd4339c4a54a2c56fe, ASSIGN}] 2024-11-09T02:21:20,479 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aad4fc6835c8f9bd4339c4a54a2c56fe, ASSIGN 2024-11-09T02:21:20,481 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aad4fc6835c8f9bd4339c4a54a2c56fe, ASSIGN; state=OFFLINE, location=53f286d3af3b,46079,1731118875223; forceNewPlan=false, retain=false 2024-11-09T02:21:20,633 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=aad4fc6835c8f9bd4339c4a54a2c56fe, regionState=OPENING, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:20,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aad4fc6835c8f9bd4339c4a54a2c56fe, ASSIGN because future has completed 2024-11-09T02:21:20,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223}] 2024-11-09T02:21:20,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T02:21:20,798 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:20,798 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:21:20,799 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:20,799 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:21:20,799 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:20,799 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:20,802 INFO [StoreOpener-aad4fc6835c8f9bd4339c4a54a2c56fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:20,804 INFO [StoreOpener-aad4fc6835c8f9bd4339c4a54a2c56fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aad4fc6835c8f9bd4339c4a54a2c56fe columnFamilyName A 2024-11-09T02:21:20,805 DEBUG [StoreOpener-aad4fc6835c8f9bd4339c4a54a2c56fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:20,806 INFO [StoreOpener-aad4fc6835c8f9bd4339c4a54a2c56fe-1 {}] regionserver.HStore(327): Store=aad4fc6835c8f9bd4339c4a54a2c56fe/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:21:20,806 INFO [StoreOpener-aad4fc6835c8f9bd4339c4a54a2c56fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:20,808 INFO [StoreOpener-aad4fc6835c8f9bd4339c4a54a2c56fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aad4fc6835c8f9bd4339c4a54a2c56fe columnFamilyName B 2024-11-09T02:21:20,808 DEBUG [StoreOpener-aad4fc6835c8f9bd4339c4a54a2c56fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:20,809 INFO [StoreOpener-aad4fc6835c8f9bd4339c4a54a2c56fe-1 {}] regionserver.HStore(327): Store=aad4fc6835c8f9bd4339c4a54a2c56fe/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:21:20,809 INFO [StoreOpener-aad4fc6835c8f9bd4339c4a54a2c56fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:20,811 INFO [StoreOpener-aad4fc6835c8f9bd4339c4a54a2c56fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aad4fc6835c8f9bd4339c4a54a2c56fe columnFamilyName C 2024-11-09T02:21:20,811 DEBUG [StoreOpener-aad4fc6835c8f9bd4339c4a54a2c56fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:20,812 INFO [StoreOpener-aad4fc6835c8f9bd4339c4a54a2c56fe-1 {}] regionserver.HStore(327): Store=aad4fc6835c8f9bd4339c4a54a2c56fe/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:21:20,812 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:20,814 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:20,815 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:20,817 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:20,817 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:20,818 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T02:21:20,820 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:20,823 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:21:20,824 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened aad4fc6835c8f9bd4339c4a54a2c56fe; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69942749, jitterRate=0.042228177189826965}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T02:21:20,825 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:20,827 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for aad4fc6835c8f9bd4339c4a54a2c56fe: Running coprocessor pre-open hook at 1731118880800Writing region info on filesystem at 1731118880800Initializing all the Stores at 1731118880801 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118880801Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118880802 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118880802Cleaning up temporary data from old regions at 1731118880817 (+15 ms)Running coprocessor post-open hooks at 1731118880825 (+8 ms)Region opened successfully at 1731118880826 (+1 ms) 2024-11-09T02:21:20,829 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., pid=6, masterSystemTime=1731118880792 2024-11-09T02:21:20,832 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:20,832 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:20,833 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=aad4fc6835c8f9bd4339c4a54a2c56fe, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:20,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 because future has completed 2024-11-09T02:21:20,844 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-09T02:21:20,845 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 in 202 msec 2024-11-09T02:21:20,848 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-09T02:21:20,848 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aad4fc6835c8f9bd4339c4a54a2c56fe, ASSIGN in 367 msec 2024-11-09T02:21:20,850 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:21:20,850 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118880850"}]},"ts":"1731118880850"} 2024-11-09T02:21:20,853 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T02:21:20,854 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:21:20,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2410 sec 2024-11-09T02:21:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T02:21:21,792 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:21:21,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,796 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:21:21,796 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:21:21,798 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:21:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:21:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:21:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5edd1052, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:21:21,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:21:21,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:21,801 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:21:21,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34fab187, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,803 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:21:21,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:21:21,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:21,807 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:21,809 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T02:21:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 53f286d3af3b,45041,1731118874457 2024-11-09T02:21:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@14016810 2024-11-09T02:21:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T02:21:21,815 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T02:21:21,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,819 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:21:21,819 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:21:21,820 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:21:21,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:21:21,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:21:21,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b2da332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:21:21,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:21:21,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:21,823 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:21:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e3687d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,828 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,828 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:21:21,829 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:21:21,830 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:21:21,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:21:21,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:21:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8f5a6b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:21:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:21:21,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:21,833 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:21:21,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b383f3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,836 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,836 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:21:21,836 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:21:21,838 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:21:21,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:21:21,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:21:21,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30bed14c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:21:21,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:21:21,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:21,843 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:21:21,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1d7645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,851 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,851 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:21:21,851 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:21:21,853 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:21:21,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:21:21,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:21:21,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27ff6f12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:21:21,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:21:21,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:21,857 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:21:21,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41f810a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,861 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,861 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:21:21,861 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:21:21,862 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:21:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:21:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:21:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76d96d2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:21:21,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:21:21,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:21,865 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:21:21,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aed53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,868 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:21:21,869 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:21:21,870 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:21:21,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:21:21,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:21:21,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@201fc254, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:21:21,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:21:21,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:21,873 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:21:21,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c7a39e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,877 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:21:21,877 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:21:21,878 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:21:21,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:21:21,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:21:21,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@425be669, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:21:21,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:21:21,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:21,881 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:21:21,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ad430e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,885 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,885 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:21:21,886 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:21:21,887 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:21:21,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:21:21,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:21:21,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40d5a1a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:21:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:21:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:21,892 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:21:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@522fc2db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,899 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:21:21,899 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:21:21,900 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:21:21,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:21:21,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:21:21,902 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19dd9b22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,902 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:21:21,902 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:21:21,902 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:21,904 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:21:21,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b52a890, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:21,922 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:21:21,922 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:21:21,923 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:21:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:21:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:21:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:21:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:21,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:21,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:21,929 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:21,933 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:21:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:21:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:21:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:21:21,939 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:21,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:21:21,942 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:21,946 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:21:21,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:21:21,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:21:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:21,949 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:21:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:21:21,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:21,952 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:21,953 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:21:21,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:21:21,969 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:21:21,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:21:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:21,974 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:21:21,981 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:21:21,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:21:21,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:21,986 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:21,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:21:21,992 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:21:22,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:21:22,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:22,002 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:22,002 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:21:22,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:21:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-09T02:21:22,018 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:21:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T02:21:22,023 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:21:22,027 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:21:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:22,106 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:21:22,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aad4fc6835c8f9bd4339c4a54a2c56fe 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-09T02:21:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T02:21:22,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T02:21:22,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:22,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:22,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:22,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:22,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:22,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/67b4068fcca6421587a38bbbe88e00eb is 50, key is test_row_0/A:col10/1731118882098/Put/seqid=0 2024-11-09T02:21:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T02:21:22,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741837_1013 (size=12001) 2024-11-09T02:21:22,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:22,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118942374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:22,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118942379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:22,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58422 deadline: 1731118942389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:22,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58354 deadline: 1731118942364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:22,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:22,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118942400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:22,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T02:21:22,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:22,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:22,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:22,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:22,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:22,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118942521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:22,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118942521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:22,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:22,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:22,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58354 deadline: 1731118942523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:22,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:22,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118942531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58422 deadline: 1731118942522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:22,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:22,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:22,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T02:21:22,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:22,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:22,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:22,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:22,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T02:21:22,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118942741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:22,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:22,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118942742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:22,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:22,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58354 deadline: 1731118942750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:22,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:22,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118942752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:22,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58422 deadline: 1731118942751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:22,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:22,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:22,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:22,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/67b4068fcca6421587a38bbbe88e00eb 2024-11-09T02:21:22,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T02:21:22,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:22,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:22,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:22,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:22,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:22,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/f84d955ed2ce40b69754de6a9b2b3cc8 is 50, key is test_row_0/B:col10/1731118882098/Put/seqid=0 2024-11-09T02:21:22,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T02:21:22,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:22,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:22,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:22,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:22,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:23,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741838_1014 (size=12001) 2024-11-09T02:21:23,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118943050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:23,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:23,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118943050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:23,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:23,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58354 deadline: 1731118943061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:23,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118943071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:23,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:23,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58422 deadline: 1731118943071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:23,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:23,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T02:21:23,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:23,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:23,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:23,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:23,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T02:21:23,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T02:21:23,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:23,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:23,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:23,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:23,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:23,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/f84d955ed2ce40b69754de6a9b2b3cc8 2024-11-09T02:21:23,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T02:21:23,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:23,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:23,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:23,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:23,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:23,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/f6475ea7cf934960868b5fa942b61524 is 50, key is test_row_0/C:col10/1731118882098/Put/seqid=0 2024-11-09T02:21:23,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741839_1015 (size=12001) 2024-11-09T02:21:23,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/f6475ea7cf934960868b5fa942b61524 2024-11-09T02:21:23,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/67b4068fcca6421587a38bbbe88e00eb as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/67b4068fcca6421587a38bbbe88e00eb 2024-11-09T02:21:23,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/67b4068fcca6421587a38bbbe88e00eb, entries=150, sequenceid=15, filesize=11.7 K 2024-11-09T02:21:23,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/f84d955ed2ce40b69754de6a9b2b3cc8 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/f84d955ed2ce40b69754de6a9b2b3cc8 2024-11-09T02:21:23,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/f84d955ed2ce40b69754de6a9b2b3cc8, entries=150, sequenceid=15, filesize=11.7 K 2024-11-09T02:21:23,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/f6475ea7cf934960868b5fa942b61524 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/f6475ea7cf934960868b5fa942b61524 2024-11-09T02:21:23,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T02:21:23,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:23,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:23,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:23,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:23,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:23,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:23,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/f6475ea7cf934960868b5fa942b61524, entries=150, sequenceid=15, filesize=11.7 K 2024-11-09T02:21:23,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for aad4fc6835c8f9bd4339c4a54a2c56fe in 1513ms, sequenceid=15, compaction requested=false 2024-11-09T02:21:23,629 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-09T02:21:23,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:23,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:23,646 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:21:23,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aad4fc6835c8f9bd4339c4a54a2c56fe 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-09T02:21:23,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/28ef4f78a24b4ebe87ca4c5351924545 is 50, key is test_row_0/A:col10/1731118883638/Put/seqid=0 2024-11-09T02:21:23,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741840_1016 (size=12001) 2024-11-09T02:21:23,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/28ef4f78a24b4ebe87ca4c5351924545 2024-11-09T02:21:23,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58422 deadline: 1731118943706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:23,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:23,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58354 deadline: 1731118943719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:23,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118943726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:23,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:23,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118943730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:23,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:23,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118943732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:23,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:23,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T02:21:23,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:23,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:23,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:23,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:23,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:23,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/3a7bfe55802d481589145ee90edb6ce4 is 50, key is test_row_0/B:col10/1731118883638/Put/seqid=0 2024-11-09T02:21:23,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741841_1017 (size=12001) 2024-11-09T02:21:23,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/3a7bfe55802d481589145ee90edb6ce4 2024-11-09T02:21:23,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58422 deadline: 1731118943841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:23,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:23,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118943852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:23,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118943854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:23,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:23,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58354 deadline: 1731118943861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:23,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118943861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:23,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:23,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:23,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:23,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/59a3c3e9b11e43d19e54750a4293caa8 is 50, key is test_row_0/C:col10/1731118883638/Put/seqid=0 2024-11-09T02:21:23,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T02:21:23,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:23,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:23,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:23,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:23,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:23,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741842_1018 (size=12001) 2024-11-09T02:21:23,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/59a3c3e9b11e43d19e54750a4293caa8 2024-11-09T02:21:23,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/28ef4f78a24b4ebe87ca4c5351924545 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/28ef4f78a24b4ebe87ca4c5351924545 2024-11-09T02:21:23,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/28ef4f78a24b4ebe87ca4c5351924545, entries=150, sequenceid=39, filesize=11.7 K 2024-11-09T02:21:23,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/3a7bfe55802d481589145ee90edb6ce4 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/3a7bfe55802d481589145ee90edb6ce4 2024-11-09T02:21:24,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/3a7bfe55802d481589145ee90edb6ce4, entries=150, sequenceid=39, filesize=11.7 K 2024-11-09T02:21:24,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/59a3c3e9b11e43d19e54750a4293caa8 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/59a3c3e9b11e43d19e54750a4293caa8 2024-11-09T02:21:24,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/59a3c3e9b11e43d19e54750a4293caa8, entries=150, sequenceid=39, filesize=11.7 K 2024-11-09T02:21:24,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for aad4fc6835c8f9bd4339c4a54a2c56fe in 390ms, sequenceid=39, compaction requested=false 2024-11-09T02:21:24,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:24,066 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:21:24,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aad4fc6835c8f9bd4339c4a54a2c56fe 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-09T02:21:24,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T02:21:24,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:24,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:24,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:24,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:24,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:24,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/660e55d3404c41f1a6eb68490419a7c9 is 50, key is test_row_0/A:col10/1731118884061/Put/seqid=0 2024-11-09T02:21:24,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741843_1019 (size=14341) 2024-11-09T02:21:24,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/660e55d3404c41f1a6eb68490419a7c9 2024-11-09T02:21:24,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T02:21:24,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/cffa16b74f224f64be6057b0a3a95075 is 50, key is test_row_0/B:col10/1731118884061/Put/seqid=0 2024-11-09T02:21:24,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:24,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118944195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:24,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:24,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:24,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58422 deadline: 1731118944196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:24,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:24,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58354 deadline: 1731118944200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:24,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:24,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:24,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118944200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:24,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118944206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:24,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T02:21:24,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:24,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:24,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:24,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:24,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:24,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741844_1020 (size=12001) 2024-11-09T02:21:24,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/cffa16b74f224f64be6057b0a3a95075 2024-11-09T02:21:24,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:24,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58422 deadline: 1731118944332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:24,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:24,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58354 deadline: 1731118944332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:24,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:24,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:24,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118944333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:24,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:24,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:24,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:24,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118944333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:24,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:24,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118944333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:24,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:24,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/1159651d53b4478d800acf8b8e595c57 is 50, key is test_row_0/C:col10/1731118884061/Put/seqid=0 2024-11-09T02:21:24,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T02:21:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:24,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:24,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:24,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:24,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741845_1021 (size=12001) 2024-11-09T02:21:24,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T02:21:24,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:24,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:24,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:24,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:24,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:24,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:24,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58422 deadline: 1731118944550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:24,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:24,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118944562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:24,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118944562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:24,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58354 deadline: 1731118944562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:24,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:24,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118944562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:24,697 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:21:24,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T02:21:24,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:24,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:24,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:24,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:24,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:24,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/1159651d53b4478d800acf8b8e595c57 2024-11-09T02:21:24,836 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-09T02:21:24,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/660e55d3404c41f1a6eb68490419a7c9 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/660e55d3404c41f1a6eb68490419a7c9 2024-11-09T02:21:24,853 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-09T02:21:24,855 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-09T02:21:24,857 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T02:21:24,857 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-09T02:21:24,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T02:21:24,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:24,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:24,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:24,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:24,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:24,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/660e55d3404c41f1a6eb68490419a7c9, entries=200, sequenceid=53, filesize=14.0 K 2024-11-09T02:21:24,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58422 deadline: 1731118944860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:24,858 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-09T02:21:24,870 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-09T02:21:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:24,872 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-09T02:21:24,872 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-09T02:21:24,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/cffa16b74f224f64be6057b0a3a95075 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/cffa16b74f224f64be6057b0a3a95075 2024-11-09T02:21:24,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/cffa16b74f224f64be6057b0a3a95075, entries=150, sequenceid=53, filesize=11.7 K 2024-11-09T02:21:24,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:24,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/1159651d53b4478d800acf8b8e595c57 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/1159651d53b4478d800acf8b8e595c57 2024-11-09T02:21:24,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58354 deadline: 1731118944882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:24,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118944882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:24,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:24,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118944882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:24,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:24,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:24,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118944881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:24,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:24,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:24,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/1159651d53b4478d800acf8b8e595c57, entries=150, sequenceid=53, filesize=11.7 K 2024-11-09T02:21:24,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for aad4fc6835c8f9bd4339c4a54a2c56fe in 846ms, sequenceid=53, compaction requested=true 2024-11-09T02:21:24,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:24,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:21:24,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:24,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:21:24,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:24,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:21:24,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:21:24,933 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:24,937 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:24,938 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:24,941 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/B is initiating minor compaction (all files) 2024-11-09T02:21:24,941 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/B in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:24,942 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/f84d955ed2ce40b69754de6a9b2b3cc8, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/3a7bfe55802d481589145ee90edb6ce4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/cffa16b74f224f64be6057b0a3a95075] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=35.2 K 2024-11-09T02:21:24,944 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting f84d955ed2ce40b69754de6a9b2b3cc8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731118882014 2024-11-09T02:21:24,945 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a7bfe55802d481589145ee90edb6ce4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731118882389 2024-11-09T02:21:24,950 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:24,950 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting cffa16b74f224f64be6057b0a3a95075, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731118883725 2024-11-09T02:21:24,950 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/A is initiating minor compaction (all files) 2024-11-09T02:21:24,951 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/A in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:24,951 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/67b4068fcca6421587a38bbbe88e00eb, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/28ef4f78a24b4ebe87ca4c5351924545, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/660e55d3404c41f1a6eb68490419a7c9] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=37.4 K 2024-11-09T02:21:24,952 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67b4068fcca6421587a38bbbe88e00eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731118882014 2024-11-09T02:21:24,954 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28ef4f78a24b4ebe87ca4c5351924545, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731118882389 2024-11-09T02:21:24,955 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 660e55d3404c41f1a6eb68490419a7c9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731118883688 2024-11-09T02:21:25,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T02:21:25,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:25,026 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing aad4fc6835c8f9bd4339c4a54a2c56fe 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-09T02:21:25,031 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#A#compaction#9 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:25,032 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/e825b14668f84815adf9f3737cb83bbd is 50, key is test_row_0/A:col10/1731118884061/Put/seqid=0 2024-11-09T02:21:25,037 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#B#compaction#10 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:25,038 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/28f2d0655ce1487abf8e5d67a15d63de is 50, key is test_row_0/B:col10/1731118884061/Put/seqid=0 2024-11-09T02:21:25,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/21e4b9250c474c9eb9aa69554d8e062a is 50, key is test_row_0/A:col10/1731118884162/Put/seqid=0 2024-11-09T02:21:25,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741846_1022 (size=12104) 2024-11-09T02:21:25,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741847_1023 (size=12104) 2024-11-09T02:21:25,146 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/28f2d0655ce1487abf8e5d67a15d63de as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/28f2d0655ce1487abf8e5d67a15d63de 2024-11-09T02:21:25,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741848_1024 (size=12001) 2024-11-09T02:21:25,149 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/21e4b9250c474c9eb9aa69554d8e062a 2024-11-09T02:21:25,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/453a05298c4f474594353bbd05fab6f8 is 50, key is test_row_0/B:col10/1731118884162/Put/seqid=0 2024-11-09T02:21:25,191 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/B of aad4fc6835c8f9bd4339c4a54a2c56fe into 28f2d0655ce1487abf8e5d67a15d63de(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:25,191 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:25,194 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/B, priority=13, startTime=1731118884931; duration=0sec 2024-11-09T02:21:25,206 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:21:25,206 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:B 2024-11-09T02:21:25,206 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:25,216 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:25,216 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/C is initiating minor compaction (all files) 2024-11-09T02:21:25,216 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/C in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:25,217 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/f6475ea7cf934960868b5fa942b61524, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/59a3c3e9b11e43d19e54750a4293caa8, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/1159651d53b4478d800acf8b8e595c57] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=35.2 K 2024-11-09T02:21:25,219 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting f6475ea7cf934960868b5fa942b61524, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731118882014 2024-11-09T02:21:25,225 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 59a3c3e9b11e43d19e54750a4293caa8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731118882389 2024-11-09T02:21:25,228 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 1159651d53b4478d800acf8b8e595c57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731118883725 2024-11-09T02:21:25,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741849_1025 (size=12001) 2024-11-09T02:21:25,246 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/453a05298c4f474594353bbd05fab6f8 2024-11-09T02:21:25,278 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#C#compaction#13 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:25,279 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/aa09c79478cb4179b4fe6ac78b01099d is 50, key is test_row_0/C:col10/1731118884061/Put/seqid=0 2024-11-09T02:21:25,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/a17e61f611454578a4272576a50c13c4 is 50, key is test_row_0/C:col10/1731118884162/Put/seqid=0 2024-11-09T02:21:25,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741850_1026 (size=12104) 2024-11-09T02:21:25,364 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/aa09c79478cb4179b4fe6ac78b01099d as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/aa09c79478cb4179b4fe6ac78b01099d 2024-11-09T02:21:25,383 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/C of aad4fc6835c8f9bd4339c4a54a2c56fe into aa09c79478cb4179b4fe6ac78b01099d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:25,383 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:25,383 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/C, priority=13, startTime=1731118884931; duration=0sec 2024-11-09T02:21:25,383 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:25,383 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:C 2024-11-09T02:21:25,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741851_1027 (size=12001) 2024-11-09T02:21:25,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:25,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:25,395 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/a17e61f611454578a4272576a50c13c4 2024-11-09T02:21:25,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/21e4b9250c474c9eb9aa69554d8e062a as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/21e4b9250c474c9eb9aa69554d8e062a 2024-11-09T02:21:25,420 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/21e4b9250c474c9eb9aa69554d8e062a, entries=150, sequenceid=75, filesize=11.7 K 2024-11-09T02:21:25,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/453a05298c4f474594353bbd05fab6f8 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/453a05298c4f474594353bbd05fab6f8 2024-11-09T02:21:25,437 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/453a05298c4f474594353bbd05fab6f8, entries=150, sequenceid=75, filesize=11.7 K 2024-11-09T02:21:25,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/a17e61f611454578a4272576a50c13c4 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/a17e61f611454578a4272576a50c13c4 2024-11-09T02:21:25,456 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/a17e61f611454578a4272576a50c13c4, entries=150, sequenceid=75, filesize=11.7 K 2024-11-09T02:21:25,461 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=46.96 KB/48090 for aad4fc6835c8f9bd4339c4a54a2c56fe in 435ms, sequenceid=75, compaction requested=false 2024-11-09T02:21:25,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:25,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:25,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-09T02:21:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-09T02:21:25,465 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:21:25,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aad4fc6835c8f9bd4339c4a54a2c56fe 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T02:21:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:25,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-09T02:21:25,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4380 sec 2024-11-09T02:21:25,480 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.4720 sec 2024-11-09T02:21:25,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/f17353b2b62e4a609243a507e190d10d is 50, key is test_row_0/A:col10/1731118885450/Put/seqid=0 2024-11-09T02:21:25,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741852_1028 (size=16681) 2024-11-09T02:21:25,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/f17353b2b62e4a609243a507e190d10d 2024-11-09T02:21:25,567 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/e825b14668f84815adf9f3737cb83bbd as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/e825b14668f84815adf9f3737cb83bbd 2024-11-09T02:21:25,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/50184eeeab03498a908f90209353d774 is 50, key is test_row_0/B:col10/1731118885450/Put/seqid=0 2024-11-09T02:21:25,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741853_1029 (size=12001) 2024-11-09T02:21:25,622 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/A of aad4fc6835c8f9bd4339c4a54a2c56fe into e825b14668f84815adf9f3737cb83bbd(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:25,622 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:25,622 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/A, priority=13, startTime=1731118884915; duration=0sec 2024-11-09T02:21:25,622 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:25,622 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:A 2024-11-09T02:21:25,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:25,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118945627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:25,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:25,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118945628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:25,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:25,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:25,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:25,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58422 deadline: 1731118945641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:25,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:25,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:25,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118945648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:25,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:25,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58354 deadline: 1731118945652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:25,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:25,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118945760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:25,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:25,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:25,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118945761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:25,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58422 deadline: 1731118945761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118945761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:25,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58354 deadline: 1731118945761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:25,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:25,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118945972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:25,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:25,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:25,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58422 deadline: 1731118945982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:25,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:25,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118945982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:25,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:25,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118945982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:25,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58354 deadline: 1731118945985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:26,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:26,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/50184eeeab03498a908f90209353d774 2024-11-09T02:21:26,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/dc3615df843a47e893aad8665efa9e79 is 50, key is test_row_0/C:col10/1731118885450/Put/seqid=0 2024-11-09T02:21:26,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741854_1030 (size=12001) 2024-11-09T02:21:26,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/dc3615df843a47e893aad8665efa9e79 2024-11-09T02:21:26,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/f17353b2b62e4a609243a507e190d10d as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/f17353b2b62e4a609243a507e190d10d 2024-11-09T02:21:26,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/f17353b2b62e4a609243a507e190d10d, entries=250, sequenceid=88, filesize=16.3 K 2024-11-09T02:21:26,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/50184eeeab03498a908f90209353d774 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/50184eeeab03498a908f90209353d774 2024-11-09T02:21:26,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/50184eeeab03498a908f90209353d774, entries=150, sequenceid=88, filesize=11.7 K 2024-11-09T02:21:26,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/dc3615df843a47e893aad8665efa9e79 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/dc3615df843a47e893aad8665efa9e79 2024-11-09T02:21:26,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/dc3615df843a47e893aad8665efa9e79, entries=150, sequenceid=88, filesize=11.7 K 2024-11-09T02:21:26,151 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for aad4fc6835c8f9bd4339c4a54a2c56fe in 686ms, sequenceid=88, compaction requested=true 2024-11-09T02:21:26,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:26,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:21:26,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:26,152 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:26,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:21:26,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:21:26,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:21:26,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T02:21:26,153 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:26,154 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:26,154 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:26,154 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/B is initiating minor compaction (all files) 2024-11-09T02:21:26,154 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/A is initiating minor compaction (all files) 2024-11-09T02:21:26,155 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/B in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:26,155 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/A in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:26,155 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/28f2d0655ce1487abf8e5d67a15d63de, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/453a05298c4f474594353bbd05fab6f8, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/50184eeeab03498a908f90209353d774] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=35.3 K 2024-11-09T02:21:26,155 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/e825b14668f84815adf9f3737cb83bbd, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/21e4b9250c474c9eb9aa69554d8e062a, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/f17353b2b62e4a609243a507e190d10d] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=39.8 K 2024-11-09T02:21:26,156 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 28f2d0655ce1487abf8e5d67a15d63de, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731118883725 2024-11-09T02:21:26,156 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting e825b14668f84815adf9f3737cb83bbd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731118883725 2024-11-09T02:21:26,157 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21e4b9250c474c9eb9aa69554d8e062a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731118884162 2024-11-09T02:21:26,157 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting f17353b2b62e4a609243a507e190d10d, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731118885416 2024-11-09T02:21:26,160 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 453a05298c4f474594353bbd05fab6f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731118884162 2024-11-09T02:21:26,161 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 50184eeeab03498a908f90209353d774, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731118885418 2024-11-09T02:21:26,182 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#A#compaction#18 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:26,183 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/1e4e9ef8b33b4d4a87af28a66b082ad1 is 50, key is test_row_0/A:col10/1731118885450/Put/seqid=0 2024-11-09T02:21:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T02:21:26,190 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:21:26,191 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:21:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-09T02:21:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T02:21:26,198 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:21:26,201 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:21:26,201 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:21:26,202 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:26,203 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/797df0795dee49bbb6c29e64878c0051 is 50, key is test_row_0/B:col10/1731118885450/Put/seqid=0 2024-11-09T02:21:26,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741855_1031 (size=12207) 2024-11-09T02:21:26,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741856_1032 (size=12207) 2024-11-09T02:21:26,243 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/1e4e9ef8b33b4d4a87af28a66b082ad1 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/1e4e9ef8b33b4d4a87af28a66b082ad1 2024-11-09T02:21:26,259 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/A of aad4fc6835c8f9bd4339c4a54a2c56fe into 1e4e9ef8b33b4d4a87af28a66b082ad1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:26,260 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:26,260 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/A, priority=13, startTime=1731118886152; duration=0sec 2024-11-09T02:21:26,261 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:21:26,261 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:A 2024-11-09T02:21:26,261 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:26,263 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:26,264 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/C is initiating minor compaction (all files) 2024-11-09T02:21:26,264 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/C in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:26,264 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/aa09c79478cb4179b4fe6ac78b01099d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/a17e61f611454578a4272576a50c13c4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/dc3615df843a47e893aad8665efa9e79] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=35.3 K 2024-11-09T02:21:26,265 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa09c79478cb4179b4fe6ac78b01099d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731118883725 2024-11-09T02:21:26,266 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting a17e61f611454578a4272576a50c13c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731118884162 2024-11-09T02:21:26,267 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc3615df843a47e893aad8665efa9e79, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731118885418 2024-11-09T02:21:26,292 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:21:26,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aad4fc6835c8f9bd4339c4a54a2c56fe 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-09T02:21:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:26,301 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#C#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T02:21:26,302 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/591bdcac8caf45e3b7b8718cbfbb7470 is 50, key is test_row_0/C:col10/1731118885450/Put/seqid=0 2024-11-09T02:21:26,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/fe53240e16d34361b591037a42d0665a is 50, key is test_row_0/A:col10/1731118885647/Put/seqid=0 2024-11-09T02:21:26,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118946316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:26,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:26,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118946316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:26,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:26,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58354 deadline: 1731118946325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:26,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:26,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58422 deadline: 1731118946325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:26,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118946326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:26,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:26,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741857_1033 (size=12207) 2024-11-09T02:21:26,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T02:21:26,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:26,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:26,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:26,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:26,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:26,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741858_1034 (size=14341) 2024-11-09T02:21:26,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118946431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:26,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:26,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118946440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:26,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:26,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118946451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:26,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:26,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58354 deadline: 1731118946451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:26,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58422 deadline: 1731118946451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:26,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T02:21:26,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T02:21:26,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:26,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:26,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:26,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:26,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:26,628 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/797df0795dee49bbb6c29e64878c0051 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/797df0795dee49bbb6c29e64878c0051 2024-11-09T02:21:26,645 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/B of aad4fc6835c8f9bd4339c4a54a2c56fe into 797df0795dee49bbb6c29e64878c0051(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:26,645 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:26,645 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/B, priority=13, startTime=1731118886152; duration=0sec 2024-11-09T02:21:26,645 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:26,645 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:B 2024-11-09T02:21:26,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118946650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:26,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:26,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118946660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:26,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118946660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:26,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:26,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58354 deadline: 1731118946661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:26,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T02:21:26,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:26,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:26,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:26,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:26,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:26,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58422 deadline: 1731118946670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:26,769 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/591bdcac8caf45e3b7b8718cbfbb7470 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/591bdcac8caf45e3b7b8718cbfbb7470 2024-11-09T02:21:26,781 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/C of aad4fc6835c8f9bd4339c4a54a2c56fe into 591bdcac8caf45e3b7b8718cbfbb7470(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:26,781 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:26,781 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/C, priority=13, startTime=1731118886152; duration=0sec 2024-11-09T02:21:26,782 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:26,782 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:C 2024-11-09T02:21:26,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/fe53240e16d34361b591037a42d0665a 2024-11-09T02:21:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T02:21:26,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T02:21:26,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:26,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:26,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:26,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:26,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:26,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/ef50fa4d6cb0480f9c9feda6fe803e24 is 50, key is test_row_0/B:col10/1731118885647/Put/seqid=0 2024-11-09T02:21:26,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741859_1035 (size=12001) 2024-11-09T02:21:26,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/ef50fa4d6cb0480f9c9feda6fe803e24 2024-11-09T02:21:26,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/1aae7a0b1a5646dc9e463b8ae2db7978 is 50, key is test_row_0/C:col10/1731118885647/Put/seqid=0 2024-11-09T02:21:26,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118946961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:26,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:26,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118946970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:26,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118946970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:26,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:26,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:26,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:26,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T02:21:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58354 deadline: 1731118946981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:26,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:26,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:26,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:26,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:26,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:26,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:26,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741860_1036 (size=12001) 2024-11-09T02:21:26,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/1aae7a0b1a5646dc9e463b8ae2db7978 2024-11-09T02:21:26,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58422 deadline: 1731118946990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:27,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/fe53240e16d34361b591037a42d0665a as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/fe53240e16d34361b591037a42d0665a 2024-11-09T02:21:27,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/fe53240e16d34361b591037a42d0665a, entries=200, sequenceid=117, filesize=14.0 K 2024-11-09T02:21:27,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/ef50fa4d6cb0480f9c9feda6fe803e24 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/ef50fa4d6cb0480f9c9feda6fe803e24 2024-11-09T02:21:27,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/ef50fa4d6cb0480f9c9feda6fe803e24, entries=150, sequenceid=117, filesize=11.7 K 2024-11-09T02:21:27,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/1aae7a0b1a5646dc9e463b8ae2db7978 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/1aae7a0b1a5646dc9e463b8ae2db7978 2024-11-09T02:21:27,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/1aae7a0b1a5646dc9e463b8ae2db7978, entries=150, sequenceid=117, filesize=11.7 K 2024-11-09T02:21:27,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for aad4fc6835c8f9bd4339c4a54a2c56fe in 777ms, sequenceid=117, compaction requested=false 2024-11-09T02:21:27,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:27,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T02:21:27,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:27,138 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing aad4fc6835c8f9bd4339c4a54a2c56fe 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-09T02:21:27,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/18d355d928e046a4911456e32750797c is 50, key is test_row_0/A:col10/1731118886312/Put/seqid=0 2024-11-09T02:21:27,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741861_1037 (size=12001) 2024-11-09T02:21:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T02:21:27,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:27,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118947568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:27,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:27,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:27,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118947571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:27,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:27,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:27,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58354 deadline: 1731118947580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:27,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118947577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:27,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58422 deadline: 1731118947580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:27,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:27,618 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/18d355d928e046a4911456e32750797c 2024-11-09T02:21:27,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/fd789be3533c4899852d7d5988f16bd9 is 50, key is test_row_0/B:col10/1731118886312/Put/seqid=0 2024-11-09T02:21:27,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741862_1038 (size=12001) 2024-11-09T02:21:27,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118947690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:27,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:27,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118947691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:27,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:27,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58354 deadline: 1731118947702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:27,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118947701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:27,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:27,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58422 deadline: 1731118947710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:27,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:27,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:27,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118947900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:27,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:27,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118947910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:27,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58354 deadline: 1731118947911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:27,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118947920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:27,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:27,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58422 deadline: 1731118947920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:27,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:27,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:28,064 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/fd789be3533c4899852d7d5988f16bd9 2024-11-09T02:21:28,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/efa69af4df7743a1a2f4bc5a2ea59d5b is 50, key is test_row_0/C:col10/1731118886312/Put/seqid=0 2024-11-09T02:21:28,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741863_1039 (size=12001) 2024-11-09T02:21:28,121 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/efa69af4df7743a1a2f4bc5a2ea59d5b 2024-11-09T02:21:28,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/18d355d928e046a4911456e32750797c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/18d355d928e046a4911456e32750797c 2024-11-09T02:21:28,144 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/18d355d928e046a4911456e32750797c, entries=150, sequenceid=128, filesize=11.7 K 2024-11-09T02:21:28,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/fd789be3533c4899852d7d5988f16bd9 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/fd789be3533c4899852d7d5988f16bd9 2024-11-09T02:21:28,155 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/fd789be3533c4899852d7d5988f16bd9, entries=150, sequenceid=128, filesize=11.7 K 2024-11-09T02:21:28,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/efa69af4df7743a1a2f4bc5a2ea59d5b as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/efa69af4df7743a1a2f4bc5a2ea59d5b 2024-11-09T02:21:28,168 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/efa69af4df7743a1a2f4bc5a2ea59d5b, entries=150, sequenceid=128, filesize=11.7 K 2024-11-09T02:21:28,171 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for aad4fc6835c8f9bd4339c4a54a2c56fe in 1033ms, sequenceid=128, compaction requested=true 2024-11-09T02:21:28,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:28,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:28,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-09T02:21:28,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-09T02:21:28,178 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-09T02:21:28,178 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9730 sec 2024-11-09T02:21:28,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.9880 sec 2024-11-09T02:21:28,214 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:21:28,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aad4fc6835c8f9bd4339c4a54a2c56fe 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-09T02:21:28,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:28,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:28,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58422 deadline: 1731118948230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:28,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/b8a7bf1cd8004ddea168819cb1c1bcb2 is 50, key is test_row_0/A:col10/1731118888213/Put/seqid=0 2024-11-09T02:21:28,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:28,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:28,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58354 deadline: 1731118948231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:28,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:28,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118948231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:28,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:28,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:28,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118948232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:28,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:28,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:28,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118948238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:28,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:28,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741864_1040 (size=14541) 2024-11-09T02:21:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T02:21:28,350 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:21:28,351 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:21:28,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118948350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:28,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:28,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:28,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118948350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:28,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:28,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:28,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118948351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:28,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-09T02:21:28,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T02:21:28,360 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:21:28,361 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:21:28,361 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:21:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T02:21:28,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T02:21:28,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:28,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:28,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:28,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:28,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:28,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118948560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:28,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:28,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118948560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:28,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:28,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:28,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118948560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:28,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:28,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T02:21:28,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T02:21:28,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:28,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:28,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:28,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:28,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/b8a7bf1cd8004ddea168819cb1c1bcb2 2024-11-09T02:21:28,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/900e2f3e629040779397f1c43cbaf83c is 50, key is test_row_0/B:col10/1731118888213/Put/seqid=0 2024-11-09T02:21:28,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58422 deadline: 1731118948740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:28,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:28,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58354 deadline: 1731118948765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:28,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:28,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741865_1041 (size=12151) 2024-11-09T02:21:28,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/900e2f3e629040779397f1c43cbaf83c 2024-11-09T02:21:28,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/85f2d4a2f325444faae15a5f06f2e3aa is 50, key is test_row_0/C:col10/1731118888213/Put/seqid=0 2024-11-09T02:21:28,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T02:21:28,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:28,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:28,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:28,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:28,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:28,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:28,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741866_1042 (size=12151) 2024-11-09T02:21:28,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:28,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118948870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118948870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:28,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:28,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118948870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:28,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:28,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:28,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:28,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:28,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T02:21:28,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T02:21:28,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:28,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:28,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:28,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:28,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:28,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:29,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T02:21:29,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:29,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:29,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:29,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:29,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:29,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/85f2d4a2f325444faae15a5f06f2e3aa 2024-11-09T02:21:29,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/b8a7bf1cd8004ddea168819cb1c1bcb2 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/b8a7bf1cd8004ddea168819cb1c1bcb2 2024-11-09T02:21:29,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/b8a7bf1cd8004ddea168819cb1c1bcb2, entries=200, sequenceid=157, filesize=14.2 K 2024-11-09T02:21:29,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/900e2f3e629040779397f1c43cbaf83c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/900e2f3e629040779397f1c43cbaf83c 2024-11-09T02:21:29,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/900e2f3e629040779397f1c43cbaf83c, entries=150, sequenceid=157, filesize=11.9 K 2024-11-09T02:21:29,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/85f2d4a2f325444faae15a5f06f2e3aa as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/85f2d4a2f325444faae15a5f06f2e3aa 2024-11-09T02:21:29,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/85f2d4a2f325444faae15a5f06f2e3aa, entries=150, sequenceid=157, filesize=11.9 K 2024-11-09T02:21:29,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for aad4fc6835c8f9bd4339c4a54a2c56fe in 1076ms, sequenceid=157, compaction requested=true 2024-11-09T02:21:29,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:29,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:21:29,292 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T02:21:29,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:29,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:21:29,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:21:29,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:21:29,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T02:21:29,294 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T02:21:29,296 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T02:21:29,296 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T02:21:29,296 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/B is initiating minor compaction (all files) 2024-11-09T02:21:29,296 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/A is initiating minor compaction (all files) 2024-11-09T02:21:29,296 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/B in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:29,296 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/A in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:29,296 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/797df0795dee49bbb6c29e64878c0051, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/ef50fa4d6cb0480f9c9feda6fe803e24, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/fd789be3533c4899852d7d5988f16bd9, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/900e2f3e629040779397f1c43cbaf83c] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=47.2 K 2024-11-09T02:21:29,296 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/1e4e9ef8b33b4d4a87af28a66b082ad1, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/fe53240e16d34361b591037a42d0665a, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/18d355d928e046a4911456e32750797c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/b8a7bf1cd8004ddea168819cb1c1bcb2] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=51.8 K 2024-11-09T02:21:29,297 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 797df0795dee49bbb6c29e64878c0051, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731118885418 2024-11-09T02:21:29,297 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e4e9ef8b33b4d4a87af28a66b082ad1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731118885418 2024-11-09T02:21:29,298 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting ef50fa4d6cb0480f9c9feda6fe803e24, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731118885577 2024-11-09T02:21:29,298 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe53240e16d34361b591037a42d0665a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731118885577 2024-11-09T02:21:29,299 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting fd789be3533c4899852d7d5988f16bd9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731118886312 2024-11-09T02:21:29,299 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18d355d928e046a4911456e32750797c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731118886312 2024-11-09T02:21:29,300 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 900e2f3e629040779397f1c43cbaf83c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731118887567 2024-11-09T02:21:29,300 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8a7bf1cd8004ddea168819cb1c1bcb2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731118887567 2024-11-09T02:21:29,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T02:21:29,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:29,308 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing aad4fc6835c8f9bd4339c4a54a2c56fe 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-09T02:21:29,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/4ec577bbd5274abe86be24b3b923dbf1 is 50, key is test_row_0/A:col10/1731118888234/Put/seqid=0 2024-11-09T02:21:29,329 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#A#compaction#32 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:29,329 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#B#compaction#31 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:29,331 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/6ceb74479a71458ab34dcb3f48a5d2af is 50, key is test_row_0/B:col10/1731118888213/Put/seqid=0 2024-11-09T02:21:29,331 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/0c98da9fc29f474787e8d994dd30f67e is 50, key is test_row_0/A:col10/1731118888213/Put/seqid=0 2024-11-09T02:21:29,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741867_1043 (size=12151) 2024-11-09T02:21:29,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741868_1044 (size=12493) 2024-11-09T02:21:29,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:29,419 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/4ec577bbd5274abe86be24b3b923dbf1 2024-11-09T02:21:29,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741869_1045 (size=12493) 2024-11-09T02:21:29,431 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/6ceb74479a71458ab34dcb3f48a5d2af as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/6ceb74479a71458ab34dcb3f48a5d2af 2024-11-09T02:21:29,447 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/B of aad4fc6835c8f9bd4339c4a54a2c56fe into 6ceb74479a71458ab34dcb3f48a5d2af(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:29,447 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:29,447 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/B, priority=12, startTime=1731118889292; duration=0sec 2024-11-09T02:21:29,448 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:21:29,448 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:B 2024-11-09T02:21:29,448 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T02:21:29,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/d9a3eb0a2bfe451a94a7028ee8386258 is 50, key is test_row_0/B:col10/1731118888234/Put/seqid=0 2024-11-09T02:21:29,451 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T02:21:29,451 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/C is initiating minor compaction (all files) 2024-11-09T02:21:29,452 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/C in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:29,452 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/591bdcac8caf45e3b7b8718cbfbb7470, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/1aae7a0b1a5646dc9e463b8ae2db7978, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/efa69af4df7743a1a2f4bc5a2ea59d5b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/85f2d4a2f325444faae15a5f06f2e3aa] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=47.2 K 2024-11-09T02:21:29,453 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 591bdcac8caf45e3b7b8718cbfbb7470, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731118885418 2024-11-09T02:21:29,454 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 1aae7a0b1a5646dc9e463b8ae2db7978, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731118885577 2024-11-09T02:21:29,454 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting efa69af4df7743a1a2f4bc5a2ea59d5b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731118886312 2024-11-09T02:21:29,455 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 85f2d4a2f325444faae15a5f06f2e3aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731118887567 2024-11-09T02:21:29,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741870_1046 (size=12151) 2024-11-09T02:21:29,491 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/d9a3eb0a2bfe451a94a7028ee8386258 2024-11-09T02:21:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T02:21:29,515 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#C#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:29,516 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/915922a86d1942b2bdcdfca812683c8b is 50, key is test_row_0/C:col10/1731118888213/Put/seqid=0 2024-11-09T02:21:29,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/ab625d01979d425fa63b0da3886ecf06 is 50, key is test_row_0/C:col10/1731118888234/Put/seqid=0 2024-11-09T02:21:29,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118949542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:29,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118949548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:29,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118949550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:29,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:29,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741871_1047 (size=12493) 2024-11-09T02:21:29,598 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/915922a86d1942b2bdcdfca812683c8b as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/915922a86d1942b2bdcdfca812683c8b 2024-11-09T02:21:29,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741872_1048 (size=12151) 2024-11-09T02:21:29,619 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/C of aad4fc6835c8f9bd4339c4a54a2c56fe into 915922a86d1942b2bdcdfca812683c8b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:29,619 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:29,619 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/C, priority=12, startTime=1731118889292; duration=0sec 2024-11-09T02:21:29,620 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:29,620 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:C 2024-11-09T02:21:29,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118949660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:29,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118949680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:29,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118949680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:29,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:29,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:29,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:29,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:29,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58422 deadline: 1731118949770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:29,779 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_0' in aad4fc6835c8f9bd4339c4a54a2c56fe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:21:29,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:29,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:29,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:29,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58354 deadline: 1731118949780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:29,789 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_2' in aad4fc6835c8f9bd4339c4a54a2c56fe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:21:29,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:29,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:29,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:29,839 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/0c98da9fc29f474787e8d994dd30f67e as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/0c98da9fc29f474787e8d994dd30f67e 2024-11-09T02:21:29,849 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/A of aad4fc6835c8f9bd4339c4a54a2c56fe into 0c98da9fc29f474787e8d994dd30f67e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:29,850 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:29,850 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/A, priority=12, startTime=1731118889292; duration=0sec 2024-11-09T02:21:29,850 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:29,850 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:A 2024-11-09T02:21:29,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:29,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118949870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:29,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:29,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:29,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118949890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:29,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:29,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:29,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:29,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:29,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118949890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:29,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:29,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:29,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:30,004 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/ab625d01979d425fa63b0da3886ecf06 2024-11-09T02:21:30,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/4ec577bbd5274abe86be24b3b923dbf1 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/4ec577bbd5274abe86be24b3b923dbf1 2024-11-09T02:21:30,027 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/4ec577bbd5274abe86be24b3b923dbf1, entries=150, sequenceid=164, filesize=11.9 K 2024-11-09T02:21:30,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/d9a3eb0a2bfe451a94a7028ee8386258 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/d9a3eb0a2bfe451a94a7028ee8386258 2024-11-09T02:21:30,038 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/d9a3eb0a2bfe451a94a7028ee8386258, entries=150, sequenceid=164, filesize=11.9 K 2024-11-09T02:21:30,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/ab625d01979d425fa63b0da3886ecf06 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/ab625d01979d425fa63b0da3886ecf06 2024-11-09T02:21:30,053 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/ab625d01979d425fa63b0da3886ecf06, entries=150, sequenceid=164, filesize=11.9 K 2024-11-09T02:21:30,055 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=181.14 KB/185490 for aad4fc6835c8f9bd4339c4a54a2c56fe in 747ms, sequenceid=164, compaction requested=false 2024-11-09T02:21:30,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:30,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:30,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-09T02:21:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-09T02:21:30,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-09T02:21:30,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6970 sec 2024-11-09T02:21:30,071 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.7170 sec 2024-11-09T02:21:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:30,204 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:21:30,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aad4fc6835c8f9bd4339c4a54a2c56fe 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-09T02:21:30,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:30,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118950206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:30,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/4378d5ddbfd64047b5d3af53f7212be1 is 50, key is test_row_0/A:col10/1731118890192/Put/seqid=0 2024-11-09T02:21:30,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118950211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:30,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118950213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:30,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:30,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741873_1049 (size=14541) 2024-11-09T02:21:30,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/4378d5ddbfd64047b5d3af53f7212be1 2024-11-09T02:21:30,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/6d31eca5654d44c1ade940a8bdd36d82 is 50, key is test_row_0/B:col10/1731118890192/Put/seqid=0 2024-11-09T02:21:30,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741874_1050 (size=12151) 2024-11-09T02:21:30,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/6d31eca5654d44c1ade940a8bdd36d82 2024-11-09T02:21:30,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118950320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:30,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:30,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118950320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:30,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:30,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118950320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:30,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:30,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/16873e7b44f64cd5850a010084a41617 is 50, key is test_row_0/C:col10/1731118890192/Put/seqid=0 2024-11-09T02:21:30,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741875_1051 (size=12151) 2024-11-09T02:21:30,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/16873e7b44f64cd5850a010084a41617 2024-11-09T02:21:30,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/4378d5ddbfd64047b5d3af53f7212be1 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/4378d5ddbfd64047b5d3af53f7212be1 2024-11-09T02:21:30,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/4378d5ddbfd64047b5d3af53f7212be1, entries=200, sequenceid=198, filesize=14.2 K 2024-11-09T02:21:30,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/6d31eca5654d44c1ade940a8bdd36d82 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/6d31eca5654d44c1ade940a8bdd36d82 2024-11-09T02:21:30,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/6d31eca5654d44c1ade940a8bdd36d82, entries=150, sequenceid=198, filesize=11.9 K 2024-11-09T02:21:30,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/16873e7b44f64cd5850a010084a41617 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/16873e7b44f64cd5850a010084a41617 2024-11-09T02:21:30,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/16873e7b44f64cd5850a010084a41617, entries=150, sequenceid=198, filesize=11.9 K 2024-11-09T02:21:30,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for aad4fc6835c8f9bd4339c4a54a2c56fe in 237ms, sequenceid=198, compaction requested=true 2024-11-09T02:21:30,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:30,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:21:30,441 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:30,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:30,442 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:30,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:21:30,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:30,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:21:30,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:21:30,445 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:30,445 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/A is initiating minor compaction (all files) 2024-11-09T02:21:30,445 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:30,445 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/A in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:30,445 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/B is initiating minor compaction (all files) 2024-11-09T02:21:30,445 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/0c98da9fc29f474787e8d994dd30f67e, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/4ec577bbd5274abe86be24b3b923dbf1, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/4378d5ddbfd64047b5d3af53f7212be1] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=38.3 K 2024-11-09T02:21:30,445 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/B in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:30,445 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/6ceb74479a71458ab34dcb3f48a5d2af, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/d9a3eb0a2bfe451a94a7028ee8386258, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/6d31eca5654d44c1ade940a8bdd36d82] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=35.9 K 2024-11-09T02:21:30,446 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c98da9fc29f474787e8d994dd30f67e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731118887567 2024-11-09T02:21:30,446 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ec577bbd5274abe86be24b3b923dbf1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731118888227 2024-11-09T02:21:30,446 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ceb74479a71458ab34dcb3f48a5d2af, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731118887567 2024-11-09T02:21:30,447 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4378d5ddbfd64047b5d3af53f7212be1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731118889502 2024-11-09T02:21:30,447 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting d9a3eb0a2bfe451a94a7028ee8386258, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731118888227 2024-11-09T02:21:30,448 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d31eca5654d44c1ade940a8bdd36d82, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731118889538 2024-11-09T02:21:30,484 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#A#compaction#39 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:30,485 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/092ce4f874f04e728bedda5f9656505f is 50, key is test_row_0/A:col10/1731118890192/Put/seqid=0 2024-11-09T02:21:30,494 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#B#compaction#40 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:30,495 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/819df7ff05414a02bb91325ceeff0ef6 is 50, key is test_row_0/B:col10/1731118890192/Put/seqid=0 2024-11-09T02:21:30,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T02:21:30,510 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:21:30,511 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:21:30,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-09T02:21:30,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T02:21:30,514 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:21:30,515 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:21:30,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:21:30,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741876_1052 (size=12595) 2024-11-09T02:21:30,559 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:21:30,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aad4fc6835c8f9bd4339c4a54a2c56fe 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T02:21:30,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:30,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/01d693c74af34faeb1f02f011e6174f9 is 50, key is test_row_1/A:col10/1731118890558/Put/seqid=0 2024-11-09T02:21:30,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741877_1053 (size=12595) 2024-11-09T02:21:30,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741878_1054 (size=14537) 2024-11-09T02:21:30,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T02:21:30,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/01d693c74af34faeb1f02f011e6174f9 2024-11-09T02:21:30,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/45e8b30696f6439db032a6f1593d3039 is 50, key is test_row_1/B:col10/1731118890558/Put/seqid=0 2024-11-09T02:21:30,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T02:21:30,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:30,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:30,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:30,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:30,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:30,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741879_1055 (size=9757) 2024-11-09T02:21:30,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/45e8b30696f6439db032a6f1593d3039 2024-11-09T02:21:30,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118950695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:30,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118950703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:30,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:30,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118950706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:30,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:30,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/06875fe74110448785b4cafa557261c9 is 50, key is test_row_1/C:col10/1731118890558/Put/seqid=0 2024-11-09T02:21:30,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:30,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741880_1056 (size=9757) 2024-11-09T02:21:30,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/06875fe74110448785b4cafa557261c9 2024-11-09T02:21:30,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/01d693c74af34faeb1f02f011e6174f9 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/01d693c74af34faeb1f02f011e6174f9 2024-11-09T02:21:30,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/01d693c74af34faeb1f02f011e6174f9, entries=200, sequenceid=209, filesize=14.2 K 2024-11-09T02:21:30,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/45e8b30696f6439db032a6f1593d3039 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/45e8b30696f6439db032a6f1593d3039 2024-11-09T02:21:30,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/45e8b30696f6439db032a6f1593d3039, entries=100, sequenceid=209, filesize=9.5 K 2024-11-09T02:21:30,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/06875fe74110448785b4cafa557261c9 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/06875fe74110448785b4cafa557261c9 2024-11-09T02:21:30,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/06875fe74110448785b4cafa557261c9, entries=100, sequenceid=209, filesize=9.5 K 2024-11-09T02:21:30,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for aad4fc6835c8f9bd4339c4a54a2c56fe in 255ms, sequenceid=209, compaction requested=true 2024-11-09T02:21:30,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:30,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:A, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:21:30,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T02:21:30,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:B, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:21:30,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-09T02:21:30,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:21:30,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-09T02:21:30,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:30,822 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:21:30,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aad4fc6835c8f9bd4339c4a54a2c56fe 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-09T02:21:30,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T02:21:30,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:30,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:30,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:30,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:30,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:30,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:30,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T02:21:30,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/284283e9735c443d91a55e5ef3c3a9c3 is 50, key is test_row_0/A:col10/1731118890682/Put/seqid=0 2024-11-09T02:21:30,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:30,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118950858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:30,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:30,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:30,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118950858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:30,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:30,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:30,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118950861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:30,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:30,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741881_1057 (size=14541) 2024-11-09T02:21:30,959 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/092ce4f874f04e728bedda5f9656505f as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/092ce4f874f04e728bedda5f9656505f 2024-11-09T02:21:30,969 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/A of aad4fc6835c8f9bd4339c4a54a2c56fe into 092ce4f874f04e728bedda5f9656505f(size=12.3 K), total size for store is 26.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:30,969 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:30,969 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/A, priority=13, startTime=1731118890441; duration=0sec 2024-11-09T02:21:30,970 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-09T02:21:30,970 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:A 2024-11-09T02:21:30,970 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:A 2024-11-09T02:21:30,970 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T02:21:30,972 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T02:21:30,972 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/C is initiating minor compaction (all files) 2024-11-09T02:21:30,972 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/C in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:30,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:30,972 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/915922a86d1942b2bdcdfca812683c8b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/ab625d01979d425fa63b0da3886ecf06, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/16873e7b44f64cd5850a010084a41617, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/06875fe74110448785b4cafa557261c9] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=45.5 K 2024-11-09T02:21:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118950970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:30,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:30,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118950970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:30,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:30,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118950970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:30,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:30,975 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 915922a86d1942b2bdcdfca812683c8b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731118887567 2024-11-09T02:21:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:30,975 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab625d01979d425fa63b0da3886ecf06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731118888227 2024-11-09T02:21:30,976 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16873e7b44f64cd5850a010084a41617, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731118889538 2024-11-09T02:21:30,976 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06875fe74110448785b4cafa557261c9, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731118890550 2024-11-09T02:21:30,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T02:21:30,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:30,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:30,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:30,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:30,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:30,991 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/819df7ff05414a02bb91325ceeff0ef6 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/819df7ff05414a02bb91325ceeff0ef6 2024-11-09T02:21:31,002 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/B of aad4fc6835c8f9bd4339c4a54a2c56fe into 819df7ff05414a02bb91325ceeff0ef6(size=12.3 K), total size for store is 21.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:31,002 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:31,002 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/B, priority=13, startTime=1731118890441; duration=0sec 2024-11-09T02:21:31,002 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-09T02:21:31,002 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:B 2024-11-09T02:21:31,002 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:B 2024-11-09T02:21:31,002 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-09T02:21:31,002 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-09T02:21:31,003 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-09T02:21:31,003 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. because compaction request was cancelled 2024-11-09T02:21:31,003 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:C 2024-11-09T02:21:31,003 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-09T02:21:31,004 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-09T02:21:31,004 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-09T02:21:31,005 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. because compaction request was cancelled 2024-11-09T02:21:31,005 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:B 2024-11-09T02:21:31,005 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-09T02:21:31,006 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#C#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:31,007 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-09T02:21:31,007 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-09T02:21:31,007 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. because compaction request was cancelled 2024-11-09T02:21:31,007 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/b513b3bb91ec4dbc99c32afe637f9546 is 50, key is test_row_0/C:col10/1731118890192/Put/seqid=0 2024-11-09T02:21:31,007 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:A 2024-11-09T02:21:31,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741882_1058 (size=12629) 2024-11-09T02:21:31,074 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/b513b3bb91ec4dbc99c32afe637f9546 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/b513b3bb91ec4dbc99c32afe637f9546 2024-11-09T02:21:31,092 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/C of aad4fc6835c8f9bd4339c4a54a2c56fe into b513b3bb91ec4dbc99c32afe637f9546(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:31,092 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:31,092 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/C, priority=12, startTime=1731118890816; duration=0sec 2024-11-09T02:21:31,093 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:31,093 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:C 2024-11-09T02:21:31,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T02:21:31,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:31,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:31,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:31,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:31,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T02:21:31,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118951180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:31,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:31,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118951181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:31,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:31,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:31,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118951180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:31,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:31,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/284283e9735c443d91a55e5ef3c3a9c3 2024-11-09T02:21:31,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T02:21:31,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:31,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:31,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:31,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:31,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:31,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/bd33c2a195214cd19723c68a22560d06 is 50, key is test_row_0/B:col10/1731118890682/Put/seqid=0 2024-11-09T02:21:31,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741883_1059 (size=12151) 2024-11-09T02:21:31,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/bd33c2a195214cd19723c68a22560d06 2024-11-09T02:21:31,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/a20582a74fe941838615457545d0e9cc is 50, key is test_row_0/C:col10/1731118890682/Put/seqid=0 2024-11-09T02:21:31,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741884_1060 (size=12151) 2024-11-09T02:21:31,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/a20582a74fe941838615457545d0e9cc 2024-11-09T02:21:31,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/284283e9735c443d91a55e5ef3c3a9c3 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/284283e9735c443d91a55e5ef3c3a9c3 2024-11-09T02:21:31,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/284283e9735c443d91a55e5ef3c3a9c3, entries=200, sequenceid=236, filesize=14.2 K 2024-11-09T02:21:31,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/bd33c2a195214cd19723c68a22560d06 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/bd33c2a195214cd19723c68a22560d06 2024-11-09T02:21:31,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/bd33c2a195214cd19723c68a22560d06, entries=150, sequenceid=236, filesize=11.9 K 2024-11-09T02:21:31,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/a20582a74fe941838615457545d0e9cc as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/a20582a74fe941838615457545d0e9cc 2024-11-09T02:21:31,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T02:21:31,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:31,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:31,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:31,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:31,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:31,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/a20582a74fe941838615457545d0e9cc, entries=150, sequenceid=236, filesize=11.9 K 2024-11-09T02:21:31,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for aad4fc6835c8f9bd4339c4a54a2c56fe in 646ms, sequenceid=236, compaction requested=true 2024-11-09T02:21:31,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:31,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:21:31,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:31,469 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:31,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:21:31,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:21:31,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:21:31,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T02:21:31,469 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:31,471 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41673 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:31,471 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:31,471 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/A is initiating minor compaction (all files) 2024-11-09T02:21:31,471 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/B is initiating minor compaction (all files) 2024-11-09T02:21:31,471 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/A in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:31,471 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/B in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:31,471 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/092ce4f874f04e728bedda5f9656505f, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/01d693c74af34faeb1f02f011e6174f9, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/284283e9735c443d91a55e5ef3c3a9c3] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=40.7 K 2024-11-09T02:21:31,471 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/819df7ff05414a02bb91325ceeff0ef6, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/45e8b30696f6439db032a6f1593d3039, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/bd33c2a195214cd19723c68a22560d06] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=33.7 K 2024-11-09T02:21:31,472 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 092ce4f874f04e728bedda5f9656505f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731118889538 2024-11-09T02:21:31,472 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 819df7ff05414a02bb91325ceeff0ef6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731118889538 2024-11-09T02:21:31,472 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01d693c74af34faeb1f02f011e6174f9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731118890550 2024-11-09T02:21:31,472 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 45e8b30696f6439db032a6f1593d3039, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731118890550 2024-11-09T02:21:31,473 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 284283e9735c443d91a55e5ef3c3a9c3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731118890682 2024-11-09T02:21:31,474 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting bd33c2a195214cd19723c68a22560d06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731118890682 2024-11-09T02:21:31,492 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#A#compaction#48 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:31,493 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/71b51448009b4a9a92a0a7714c4534cf is 50, key is test_row_0/A:col10/1731118890682/Put/seqid=0 2024-11-09T02:21:31,501 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:21:31,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aad4fc6835c8f9bd4339c4a54a2c56fe 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T02:21:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:31,506 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#B#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:31,507 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/ed8e341a01bb42bea2a3ba2526869d2c is 50, key is test_row_0/B:col10/1731118890682/Put/seqid=0 2024-11-09T02:21:31,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/c5dd0d14315d4ae490071b84cc0e2e3a is 50, key is test_row_0/A:col10/1731118891492/Put/seqid=0 2024-11-09T02:21:31,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741885_1061 (size=12697) 2024-11-09T02:21:31,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741886_1062 (size=12697) 2024-11-09T02:21:31,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T02:21:31,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:31,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:31,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:31,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:31,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:31,612 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/ed8e341a01bb42bea2a3ba2526869d2c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/ed8e341a01bb42bea2a3ba2526869d2c 2024-11-09T02:21:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:31,619 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/71b51448009b4a9a92a0a7714c4534cf as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/71b51448009b4a9a92a0a7714c4534cf 2024-11-09T02:21:31,621 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/B of aad4fc6835c8f9bd4339c4a54a2c56fe into ed8e341a01bb42bea2a3ba2526869d2c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:31,622 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:31,622 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/B, priority=13, startTime=1731118891469; duration=0sec 2024-11-09T02:21:31,622 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:21:31,622 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:B 2024-11-09T02:21:31,622 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-09T02:21:31,625 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-09T02:21:31,625 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-09T02:21:31,625 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. because compaction request was cancelled 2024-11-09T02:21:31,625 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:C 2024-11-09T02:21:31,629 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/A of aad4fc6835c8f9bd4339c4a54a2c56fe into 71b51448009b4a9a92a0a7714c4534cf(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:31,629 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:31,629 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/A, priority=13, startTime=1731118891469; duration=0sec 2024-11-09T02:21:31,629 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:31,629 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:A 2024-11-09T02:21:31,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118951618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:31,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:31,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118951627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:31,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:31,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741887_1063 (size=14541) 2024-11-09T02:21:31,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118951640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T02:21:31,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:31,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:31,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118951750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:31,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:31,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:31,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118951750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:31,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:31,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118951760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:31,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:31,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T02:21:31,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:31,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:31,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:31,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:31,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:31,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:31,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:31,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58422 deadline: 1731118951800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:31,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:31,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58354 deadline: 1731118951801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:31,803 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_0' in aad4fc6835c8f9bd4339c4a54a2c56fe of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:21:31,804 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_2' in aad4fc6835c8f9bd4339c4a54a2c56fe of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:21:31,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:31,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:31,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T02:21:31,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:31,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:31,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:31,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:31,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:31,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:31,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:31,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118951961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:31,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:31,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:31,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118951961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:31,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:31,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:31,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118951970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:31,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:31,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:32,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/c5dd0d14315d4ae490071b84cc0e2e3a 2024-11-09T02:21:32,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/14fbc653f16c43808085ff4004575b02 is 50, key is test_row_0/B:col10/1731118891492/Put/seqid=0 2024-11-09T02:21:32,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T02:21:32,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:32,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:32,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:32,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:32,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:32,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:32,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741888_1064 (size=12151) 2024-11-09T02:21:32,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/14fbc653f16c43808085ff4004575b02 2024-11-09T02:21:32,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/8ef68d74652847d9a30e9679a1ae38e3 is 50, key is test_row_0/C:col10/1731118891492/Put/seqid=0 2024-11-09T02:21:32,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741889_1065 (size=12151) 2024-11-09T02:21:32,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T02:21:32,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:32,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:32,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:32,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:32,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:32,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:32,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:32,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118952270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:32,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:32,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:32,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:32,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:32,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118952281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:32,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:32,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:32,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:32,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:32,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118952282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:32,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:32,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:32,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:32,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T02:21:32,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:32,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:32,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:32,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:32,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:32,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:32,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T02:21:32,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:32,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:32,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:32,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:32,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:32,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:32,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/8ef68d74652847d9a30e9679a1ae38e3 2024-11-09T02:21:32,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/c5dd0d14315d4ae490071b84cc0e2e3a as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/c5dd0d14315d4ae490071b84cc0e2e3a 2024-11-09T02:21:32,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/c5dd0d14315d4ae490071b84cc0e2e3a, entries=200, sequenceid=250, filesize=14.2 K 2024-11-09T02:21:32,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/14fbc653f16c43808085ff4004575b02 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/14fbc653f16c43808085ff4004575b02 2024-11-09T02:21:32,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/14fbc653f16c43808085ff4004575b02, entries=150, sequenceid=250, filesize=11.9 K 2024-11-09T02:21:32,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/8ef68d74652847d9a30e9679a1ae38e3 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/8ef68d74652847d9a30e9679a1ae38e3 2024-11-09T02:21:32,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/8ef68d74652847d9a30e9679a1ae38e3, entries=150, sequenceid=250, filesize=11.9 K 2024-11-09T02:21:32,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for aad4fc6835c8f9bd4339c4a54a2c56fe in 1159ms, sequenceid=250, compaction requested=true 2024-11-09T02:21:32,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:32,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:21:32,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:21:32,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:21:32,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T02:21:32,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:21:32,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-09T02:21:32,661 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:32,661 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-09T02:21:32,662 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-09T02:21:32,662 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-09T02:21:32,662 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. because compaction request was cancelled 2024-11-09T02:21:32,662 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:A 2024-11-09T02:21:32,662 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-09T02:21:32,667 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:32,667 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/C is initiating minor compaction (all files) 2024-11-09T02:21:32,667 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/C in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:32,668 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/b513b3bb91ec4dbc99c32afe637f9546, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/a20582a74fe941838615457545d0e9cc, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/8ef68d74652847d9a30e9679a1ae38e3] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=36.1 K 2024-11-09T02:21:32,668 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-09T02:21:32,669 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-09T02:21:32,669 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. because compaction request was cancelled 2024-11-09T02:21:32,669 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:B 2024-11-09T02:21:32,669 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting b513b3bb91ec4dbc99c32afe637f9546, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731118890192 2024-11-09T02:21:32,669 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting a20582a74fe941838615457545d0e9cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731118890682 2024-11-09T02:21:32,670 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ef68d74652847d9a30e9679a1ae38e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731118890837 2024-11-09T02:21:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T02:21:32,688 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#C#compaction#53 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:32,688 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/389b932d05e1435086771861cbba5ab8 is 50, key is test_row_0/C:col10/1731118891492/Put/seqid=0 2024-11-09T02:21:32,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T02:21:32,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:32,708 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing aad4fc6835c8f9bd4339c4a54a2c56fe 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T02:21:32,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/17a6430a2ced4021a62521b95e94c7d5 is 50, key is test_row_0/A:col10/1731118891601/Put/seqid=0 2024-11-09T02:21:32,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741890_1066 (size=12731) 2024-11-09T02:21:32,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741891_1067 (size=12301) 2024-11-09T02:21:32,766 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/17a6430a2ced4021a62521b95e94c7d5 2024-11-09T02:21:32,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:32,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:32,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/9e4c557ec0b645d5952ee31c23d44f30 is 50, key is test_row_0/B:col10/1731118891601/Put/seqid=0 2024-11-09T02:21:32,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741892_1068 (size=12301) 2024-11-09T02:21:32,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:32,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118952815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:32,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:32,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:32,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:32,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:32,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118952815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:32,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:32,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:32,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:32,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:32,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118952817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:32,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:32,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:32,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:32,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118952931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:32,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:32,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:32,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:32,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:32,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118952931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:32,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:32,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:32,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:32,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118952931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:33,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:33,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118953152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:33,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:33,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118953152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:33,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:33,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118953152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:33,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:33,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:33,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:33,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:33,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:33,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:33,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:33,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:33,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:33,166 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/389b932d05e1435086771861cbba5ab8 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/389b932d05e1435086771861cbba5ab8 2024-11-09T02:21:33,179 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/C of aad4fc6835c8f9bd4339c4a54a2c56fe into 389b932d05e1435086771861cbba5ab8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:33,179 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:33,179 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/C, priority=13, startTime=1731118892660; duration=0sec 2024-11-09T02:21:33,180 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:33,180 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:C 2024-11-09T02:21:33,203 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/9e4c557ec0b645d5952ee31c23d44f30 2024-11-09T02:21:33,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/fb2cc24c5c2c4207afe9e07ee19db76b is 50, key is test_row_0/C:col10/1731118891601/Put/seqid=0 2024-11-09T02:21:33,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741893_1069 (size=12301) 2024-11-09T02:21:33,310 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/fb2cc24c5c2c4207afe9e07ee19db76b 2024-11-09T02:21:33,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/17a6430a2ced4021a62521b95e94c7d5 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/17a6430a2ced4021a62521b95e94c7d5 2024-11-09T02:21:33,326 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/17a6430a2ced4021a62521b95e94c7d5, entries=150, sequenceid=277, filesize=12.0 K 2024-11-09T02:21:33,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/9e4c557ec0b645d5952ee31c23d44f30 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/9e4c557ec0b645d5952ee31c23d44f30 2024-11-09T02:21:33,335 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/9e4c557ec0b645d5952ee31c23d44f30, entries=150, sequenceid=277, filesize=12.0 K 2024-11-09T02:21:33,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/fb2cc24c5c2c4207afe9e07ee19db76b as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/fb2cc24c5c2c4207afe9e07ee19db76b 2024-11-09T02:21:33,345 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/fb2cc24c5c2c4207afe9e07ee19db76b, entries=150, sequenceid=277, filesize=12.0 K 2024-11-09T02:21:33,346 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for aad4fc6835c8f9bd4339c4a54a2c56fe in 638ms, sequenceid=277, compaction requested=true 2024-11-09T02:21:33,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:33,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:33,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-09T02:21:33,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-09T02:21:33,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-09T02:21:33,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8340 sec 2024-11-09T02:21:33,360 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.8420 sec 2024-11-09T02:21:33,479 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:21:33,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aad4fc6835c8f9bd4339c4a54a2c56fe 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T02:21:33,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:33,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/cd794c0ee6e84a20a4ac6cd3934ffec3 is 50, key is test_row_0/A:col10/1731118893477/Put/seqid=0 2024-11-09T02:21:33,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741894_1070 (size=17181) 2024-11-09T02:21:33,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/cd794c0ee6e84a20a4ac6cd3934ffec3 2024-11-09T02:21:33,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/11f3233a04894e11be54517282c479ea is 50, key is test_row_0/B:col10/1731118893477/Put/seqid=0 2024-11-09T02:21:33,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741895_1071 (size=12301) 2024-11-09T02:21:33,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/11f3233a04894e11be54517282c479ea 2024-11-09T02:21:33,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:33,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118953579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:33,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:33,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118953594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:33,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:33,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:33,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:33,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118953600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:33,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/08c51ce536a64d578e81ee34c6e59ee7 is 50, key is test_row_0/C:col10/1731118893477/Put/seqid=0 2024-11-09T02:21:33,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741896_1072 (size=12301) 2024-11-09T02:21:33,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:33,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118953700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:33,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:33,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:33,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:33,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118953710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:33,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:33,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:33,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:33,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118953710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:33,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:33,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:33,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:33,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:33,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:33,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118953920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:33,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118953920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:33,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:33,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:33,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:33,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:33,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118953920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:34,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/08c51ce536a64d578e81ee34c6e59ee7 2024-11-09T02:21:34,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/cd794c0ee6e84a20a4ac6cd3934ffec3 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/cd794c0ee6e84a20a4ac6cd3934ffec3 2024-11-09T02:21:34,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/cd794c0ee6e84a20a4ac6cd3934ffec3, entries=250, sequenceid=291, filesize=16.8 K 2024-11-09T02:21:34,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/11f3233a04894e11be54517282c479ea as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/11f3233a04894e11be54517282c479ea 2024-11-09T02:21:34,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/11f3233a04894e11be54517282c479ea, entries=150, sequenceid=291, filesize=12.0 K 2024-11-09T02:21:34,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/08c51ce536a64d578e81ee34c6e59ee7 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/08c51ce536a64d578e81ee34c6e59ee7 2024-11-09T02:21:34,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/08c51ce536a64d578e81ee34c6e59ee7, entries=150, sequenceid=291, filesize=12.0 K 2024-11-09T02:21:34,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for aad4fc6835c8f9bd4339c4a54a2c56fe in 621ms, sequenceid=291, compaction requested=true 2024-11-09T02:21:34,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:34,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:21:34,101 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T02:21:34,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:34,101 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T02:21:34,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:21:34,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:34,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:21:34,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:21:34,102 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56720 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T02:21:34,102 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/A is initiating minor compaction (all files) 2024-11-09T02:21:34,102 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/A in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:34,103 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/71b51448009b4a9a92a0a7714c4534cf, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/c5dd0d14315d4ae490071b84cc0e2e3a, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/17a6430a2ced4021a62521b95e94c7d5, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/cd794c0ee6e84a20a4ac6cd3934ffec3] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=55.4 K 2024-11-09T02:21:34,103 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T02:21:34,103 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/B is initiating minor compaction (all files) 2024-11-09T02:21:34,103 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/B in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:34,103 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/ed8e341a01bb42bea2a3ba2526869d2c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/14fbc653f16c43808085ff4004575b02, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/9e4c557ec0b645d5952ee31c23d44f30, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/11f3233a04894e11be54517282c479ea] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=48.3 K 2024-11-09T02:21:34,104 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71b51448009b4a9a92a0a7714c4534cf, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731118890682 2024-11-09T02:21:34,104 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting ed8e341a01bb42bea2a3ba2526869d2c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731118890682 2024-11-09T02:21:34,104 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5dd0d14315d4ae490071b84cc0e2e3a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731118890837 2024-11-09T02:21:34,105 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17a6430a2ced4021a62521b95e94c7d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731118891601 2024-11-09T02:21:34,105 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 14fbc653f16c43808085ff4004575b02, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731118890837 2024-11-09T02:21:34,107 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd794c0ee6e84a20a4ac6cd3934ffec3, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731118892801 2024-11-09T02:21:34,107 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e4c557ec0b645d5952ee31c23d44f30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731118891601 2024-11-09T02:21:34,107 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 11f3233a04894e11be54517282c479ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731118892812 2024-11-09T02:21:34,135 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#B#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:34,135 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/3138a97029374249a4b8e4b077a810da is 50, key is test_row_0/B:col10/1731118893477/Put/seqid=0 2024-11-09T02:21:34,144 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#A#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:34,144 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/2196ca96aa04481ea11b5d125f8165cd is 50, key is test_row_0/A:col10/1731118893477/Put/seqid=0 2024-11-09T02:21:34,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741897_1073 (size=12983) 2024-11-09T02:21:34,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741898_1074 (size=12983) 2024-11-09T02:21:34,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:34,245 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:21:34,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aad4fc6835c8f9bd4339c4a54a2c56fe 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-09T02:21:34,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/c8433e308bd84b84a26415ae77ccd7f3 is 50, key is test_row_0/A:col10/1731118893584/Put/seqid=0 2024-11-09T02:21:34,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:34,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118954267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:34,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:34,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:34,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:34,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:34,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118954271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:34,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:34,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118954271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:34,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:34,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:34,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:34,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:34,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:34,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:34,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741899_1075 (size=14741) 2024-11-09T02:21:34,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/c8433e308bd84b84a26415ae77ccd7f3 2024-11-09T02:21:34,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/4a822b37bf9d43d0816c0e73817fa304 is 50, key is test_row_0/B:col10/1731118893584/Put/seqid=0 2024-11-09T02:21:34,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741900_1076 (size=12301) 2024-11-09T02:21:34,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:34,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118954381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:34,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:34,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118954381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:34,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:34,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118954390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:34,565 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/3138a97029374249a4b8e4b077a810da as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/3138a97029374249a4b8e4b077a810da 2024-11-09T02:21:34,579 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/2196ca96aa04481ea11b5d125f8165cd as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/2196ca96aa04481ea11b5d125f8165cd 2024-11-09T02:21:34,587 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/B of aad4fc6835c8f9bd4339c4a54a2c56fe into 3138a97029374249a4b8e4b077a810da(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:34,587 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:34,587 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/B, priority=12, startTime=1731118894101; duration=0sec 2024-11-09T02:21:34,588 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:21:34,588 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:B 2024-11-09T02:21:34,588 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:34,590 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:34,591 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/C is initiating minor compaction (all files) 2024-11-09T02:21:34,592 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/C in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:34,592 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/389b932d05e1435086771861cbba5ab8, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/fb2cc24c5c2c4207afe9e07ee19db76b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/08c51ce536a64d578e81ee34c6e59ee7] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=36.5 K 2024-11-09T02:21:34,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:34,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118954590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:34,593 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 389b932d05e1435086771861cbba5ab8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731118890837 2024-11-09T02:21:34,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:34,594 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting fb2cc24c5c2c4207afe9e07ee19db76b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731118891601 2024-11-09T02:21:34,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:34,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:34,594 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 08c51ce536a64d578e81ee34c6e59ee7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731118892812 2024-11-09T02:21:34,596 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/A of aad4fc6835c8f9bd4339c4a54a2c56fe into 2196ca96aa04481ea11b5d125f8165cd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:34,597 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:34,598 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/A, priority=12, startTime=1731118894101; duration=0sec 2024-11-09T02:21:34,598 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:34,598 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:A 2024-11-09T02:21:34,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:34,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118954590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:34,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:34,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:34,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:34,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:34,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118954600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:34,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:34,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:34,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:34,626 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#C#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:34,627 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/e28835f448e3485c953ba2ad78a40e91 is 50, key is test_row_0/C:col10/1731118893477/Put/seqid=0 2024-11-09T02:21:34,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741901_1077 (size=12983) 2024-11-09T02:21:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T02:21:34,680 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:21:34,680 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:21:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-09T02:21:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T02:21:34,683 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:21:34,684 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:21:34,684 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:21:34,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/4a822b37bf9d43d0816c0e73817fa304 2024-11-09T02:21:34,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/ad024116184d413fa92479f04c165073 is 50, key is test_row_0/C:col10/1731118893584/Put/seqid=0 2024-11-09T02:21:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T02:21:34,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741902_1078 (size=12301) 2024-11-09T02:21:34,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/ad024116184d413fa92479f04c165073 2024-11-09T02:21:34,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T02:21:34,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:34,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:34,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:34,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:34,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:34,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/c8433e308bd84b84a26415ae77ccd7f3 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/c8433e308bd84b84a26415ae77ccd7f3 2024-11-09T02:21:34,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/c8433e308bd84b84a26415ae77ccd7f3, entries=200, sequenceid=315, filesize=14.4 K 2024-11-09T02:21:34,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/4a822b37bf9d43d0816c0e73817fa304 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/4a822b37bf9d43d0816c0e73817fa304 2024-11-09T02:21:34,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/4a822b37bf9d43d0816c0e73817fa304, entries=150, sequenceid=315, filesize=12.0 K 2024-11-09T02:21:34,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/ad024116184d413fa92479f04c165073 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/ad024116184d413fa92479f04c165073 2024-11-09T02:21:34,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/ad024116184d413fa92479f04c165073, entries=150, sequenceid=315, filesize=12.0 K 2024-11-09T02:21:34,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for aad4fc6835c8f9bd4339c4a54a2c56fe in 654ms, sequenceid=315, compaction requested=false 2024-11-09T02:21:34,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:34,909 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:21:34,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aad4fc6835c8f9bd4339c4a54a2c56fe 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-09T02:21:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:34,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/f3b1e0cf7d7f4e1ab73d5171d8b6a11d is 50, key is test_row_0/A:col10/1731118894267/Put/seqid=0 2024-11-09T02:21:34,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741903_1079 (size=14741) 2024-11-09T02:21:34,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:34,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118954978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118954979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:34,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118954983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:34,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:34,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:34,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:34,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T02:21:34,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:34,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:34,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:34,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:34,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:34,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T02:21:35,071 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/e28835f448e3485c953ba2ad78a40e91 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/e28835f448e3485c953ba2ad78a40e91 2024-11-09T02:21:35,082 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/C of aad4fc6835c8f9bd4339c4a54a2c56fe into e28835f448e3485c953ba2ad78a40e91(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:35,082 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:35,082 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/C, priority=13, startTime=1731118894101; duration=0sec 2024-11-09T02:21:35,082 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:35,082 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:C 2024-11-09T02:21:35,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118955090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:35,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:35,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118955090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:35,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:35,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:35,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:35,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118955091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:35,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:35,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:35,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:35,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:35,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T02:21:35,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:35,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:35,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:35,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:35,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:35,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:35,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T02:21:35,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:35,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:35,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:35,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:35,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:35,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:35,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T02:21:35,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118955310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:35,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:35,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118955310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:35,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:35,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:35,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118955310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:35,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/f3b1e0cf7d7f4e1ab73d5171d8b6a11d 2024-11-09T02:21:35,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/1ecb1b4676284af8b57ffe2c157e822c is 50, key is test_row_0/B:col10/1731118894267/Put/seqid=0 2024-11-09T02:21:35,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741904_1080 (size=12301) 2024-11-09T02:21:35,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T02:21:35,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:35,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:35,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:35,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:35,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:35,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T02:21:35,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:35,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:35,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:35,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:35,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:35,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118955620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:35,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118955620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:35,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:35,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:35,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:35,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118955620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:35,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:35,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:35,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:35,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T02:21:35,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:35,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:35,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:35,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:35,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T02:21:35,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/1ecb1b4676284af8b57ffe2c157e822c 2024-11-09T02:21:35,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58422 deadline: 1731118955830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:35,834 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_0' in aad4fc6835c8f9bd4339c4a54a2c56fe of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:21:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:35,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/59456abaa30e443c922e135ed06dfc15 is 50, key is test_row_0/C:col10/1731118894267/Put/seqid=0 2024-11-09T02:21:35,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58354 deadline: 1731118955840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:35,848 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_2' in aad4fc6835c8f9bd4339c4a54a2c56fe of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8268 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:21:35,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:35,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:35,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:35,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741905_1081 (size=12301) 2024-11-09T02:21:35,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T02:21:35,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:35,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:35,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:35,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:35,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:36,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T02:21:36,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:36,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:36,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:36,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:36,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:36,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:36,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118956131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:36,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:36,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:36,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:36,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118956133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:36,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118956134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:36,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:36,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:36,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:36,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:36,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:36,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:36,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T02:21:36,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:36,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:36,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:36,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:36,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:36,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:36,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/59456abaa30e443c922e135ed06dfc15 2024-11-09T02:21:36,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/f3b1e0cf7d7f4e1ab73d5171d8b6a11d as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/f3b1e0cf7d7f4e1ab73d5171d8b6a11d 2024-11-09T02:21:36,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/f3b1e0cf7d7f4e1ab73d5171d8b6a11d, entries=200, sequenceid=331, filesize=14.4 K 2024-11-09T02:21:36,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/1ecb1b4676284af8b57ffe2c157e822c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/1ecb1b4676284af8b57ffe2c157e822c 2024-11-09T02:21:36,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/1ecb1b4676284af8b57ffe2c157e822c, entries=150, sequenceid=331, filesize=12.0 K 2024-11-09T02:21:36,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/59456abaa30e443c922e135ed06dfc15 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/59456abaa30e443c922e135ed06dfc15 2024-11-09T02:21:36,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/59456abaa30e443c922e135ed06dfc15, entries=150, sequenceid=331, filesize=12.0 K 2024-11-09T02:21:36,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for aad4fc6835c8f9bd4339c4a54a2c56fe in 1458ms, sequenceid=331, compaction requested=true 2024-11-09T02:21:36,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:36,367 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:36,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:21:36,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:36,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:21:36,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:21:36,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:21:36,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T02:21:36,368 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:36,369 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:36,369 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/A is initiating minor compaction (all files) 2024-11-09T02:21:36,369 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/A in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:36,369 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/2196ca96aa04481ea11b5d125f8165cd, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/c8433e308bd84b84a26415ae77ccd7f3, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/f3b1e0cf7d7f4e1ab73d5171d8b6a11d] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=41.5 K 2024-11-09T02:21:36,370 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2196ca96aa04481ea11b5d125f8165cd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731118892812 2024-11-09T02:21:36,370 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:36,370 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8433e308bd84b84a26415ae77ccd7f3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731118893564 2024-11-09T02:21:36,370 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/B is initiating minor compaction (all files) 2024-11-09T02:21:36,370 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/B in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:36,370 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/3138a97029374249a4b8e4b077a810da, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/4a822b37bf9d43d0816c0e73817fa304, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/1ecb1b4676284af8b57ffe2c157e822c] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=36.7 K 2024-11-09T02:21:36,371 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 3138a97029374249a4b8e4b077a810da, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731118892812 2024-11-09T02:21:36,371 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3b1e0cf7d7f4e1ab73d5171d8b6a11d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731118894264 2024-11-09T02:21:36,372 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a822b37bf9d43d0816c0e73817fa304, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731118893564 2024-11-09T02:21:36,372 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ecb1b4676284af8b57ffe2c157e822c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731118894264 2024-11-09T02:21:36,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T02:21:36,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:36,393 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing aad4fc6835c8f9bd4339c4a54a2c56fe 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-09T02:21:36,399 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#B#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:36,399 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/bec3de834c6542f08a9b46fc1941a407 is 50, key is test_row_0/B:col10/1731118894267/Put/seqid=0 2024-11-09T02:21:36,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/e6c98b3b2bbb41f89a16a2da15ae5a1b is 50, key is test_row_0/A:col10/1731118894976/Put/seqid=0 2024-11-09T02:21:36,406 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#A#compaction#71 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:36,407 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/283368fe3c10407584d3396e9e041836 is 50, key is test_row_0/A:col10/1731118894267/Put/seqid=0 2024-11-09T02:21:36,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741907_1083 (size=12301) 2024-11-09T02:21:36,454 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/e6c98b3b2bbb41f89a16a2da15ae5a1b 2024-11-09T02:21:36,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741906_1082 (size=13085) 2024-11-09T02:21:36,471 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/bec3de834c6542f08a9b46fc1941a407 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/bec3de834c6542f08a9b46fc1941a407 2024-11-09T02:21:36,479 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/B of aad4fc6835c8f9bd4339c4a54a2c56fe into bec3de834c6542f08a9b46fc1941a407(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:36,479 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:36,479 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/B, priority=13, startTime=1731118896368; duration=0sec 2024-11-09T02:21:36,480 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:21:36,480 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:B 2024-11-09T02:21:36,480 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:36,481 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:36,482 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/C is initiating minor compaction (all files) 2024-11-09T02:21:36,482 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/C in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:36,482 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/e28835f448e3485c953ba2ad78a40e91, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/ad024116184d413fa92479f04c165073, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/59456abaa30e443c922e135ed06dfc15] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=36.7 K 2024-11-09T02:21:36,483 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting e28835f448e3485c953ba2ad78a40e91, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731118892812 2024-11-09T02:21:36,484 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting ad024116184d413fa92479f04c165073, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731118893564 2024-11-09T02:21:36,485 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 59456abaa30e443c922e135ed06dfc15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731118894264 2024-11-09T02:21:36,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741908_1084 (size=13085) 2024-11-09T02:21:36,503 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/283368fe3c10407584d3396e9e041836 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/283368fe3c10407584d3396e9e041836 2024-11-09T02:21:36,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/314db20d716f40f1883a21110a5c6191 is 50, key is test_row_0/B:col10/1731118894976/Put/seqid=0 2024-11-09T02:21:36,514 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/A of aad4fc6835c8f9bd4339c4a54a2c56fe into 283368fe3c10407584d3396e9e041836(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:36,515 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:36,515 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/A, priority=13, startTime=1731118896367; duration=0sec 2024-11-09T02:21:36,515 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:36,515 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:A 2024-11-09T02:21:36,516 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#C#compaction#73 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:36,516 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/89a52f0446e742bd8645cf9668d8e7b2 is 50, key is test_row_0/C:col10/1731118894267/Put/seqid=0 2024-11-09T02:21:36,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741909_1085 (size=12301) 2024-11-09T02:21:36,552 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/314db20d716f40f1883a21110a5c6191 2024-11-09T02:21:36,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741910_1086 (size=13085) 2024-11-09T02:21:36,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/5adc6e3199e74cae95e6e52d8c7496f5 is 50, key is test_row_0/C:col10/1731118894976/Put/seqid=0 2024-11-09T02:21:36,579 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/89a52f0446e742bd8645cf9668d8e7b2 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/89a52f0446e742bd8645cf9668d8e7b2 2024-11-09T02:21:36,586 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/C of aad4fc6835c8f9bd4339c4a54a2c56fe into 89a52f0446e742bd8645cf9668d8e7b2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:36,586 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:36,587 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/C, priority=13, startTime=1731118896368; duration=0sec 2024-11-09T02:21:36,587 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:36,587 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:C 2024-11-09T02:21:36,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741911_1087 (size=12301) 2024-11-09T02:21:36,611 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/5adc6e3199e74cae95e6e52d8c7496f5 2024-11-09T02:21:36,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/e6c98b3b2bbb41f89a16a2da15ae5a1b as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/e6c98b3b2bbb41f89a16a2da15ae5a1b 2024-11-09T02:21:36,634 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/e6c98b3b2bbb41f89a16a2da15ae5a1b, entries=150, sequenceid=354, filesize=12.0 K 2024-11-09T02:21:36,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/314db20d716f40f1883a21110a5c6191 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/314db20d716f40f1883a21110a5c6191 2024-11-09T02:21:36,647 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/314db20d716f40f1883a21110a5c6191, entries=150, sequenceid=354, filesize=12.0 K 2024-11-09T02:21:36,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/5adc6e3199e74cae95e6e52d8c7496f5 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/5adc6e3199e74cae95e6e52d8c7496f5 2024-11-09T02:21:36,666 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/5adc6e3199e74cae95e6e52d8c7496f5, entries=150, sequenceid=354, filesize=12.0 K 2024-11-09T02:21:36,668 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for aad4fc6835c8f9bd4339c4a54a2c56fe in 276ms, sequenceid=354, compaction requested=false 2024-11-09T02:21:36,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:36,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:36,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-09T02:21:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-09T02:21:36,675 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-09T02:21:36,675 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9860 sec 2024-11-09T02:21:36,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.9960 sec 2024-11-09T02:21:36,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T02:21:36,841 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:21:36,842 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:21:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-09T02:21:36,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T02:21:36,845 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:21:36,846 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:21:36,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:21:36,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T02:21:37,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-09T02:21:37,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:37,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:37,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:37,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-09T02:21:37,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-09T02:21:37,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-09T02:21:37,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 160 msec 2024-11-09T02:21:37,014 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 170 msec 2024-11-09T02:21:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T02:21:37,161 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:21:37,162 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:21:37,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-09T02:21:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T02:21:37,169 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:21:37,172 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:21:37,172 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:21:37,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:37,174 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:21:37,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aad4fc6835c8f9bd4339c4a54a2c56fe 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T02:21:37,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/d59901b62db74e06b04036595755cd62 is 50, key is test_row_0/A:col10/1731118897141/Put/seqid=0 2024-11-09T02:21:37,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741912_1088 (size=14741) 2024-11-09T02:21:37,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/d59901b62db74e06b04036595755cd62 2024-11-09T02:21:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T02:21:37,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118957269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:37,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:37,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:37,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:37,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/41c577fb7aee4c4f8ce6590cce44b793 is 50, key is test_row_0/B:col10/1731118897141/Put/seqid=0 2024-11-09T02:21:37,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118957276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:37,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:37,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:37,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:37,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118957282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:37,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:37,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:37,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:37,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T02:21:37,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:37,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:37,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:37,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:37,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:37,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741913_1089 (size=12301) 2024-11-09T02:21:37,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/41c577fb7aee4c4f8ce6590cce44b793 2024-11-09T02:21:37,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/5a7167ecb37e4814944b713932ec3ba1 is 50, key is test_row_0/C:col10/1731118897141/Put/seqid=0 2024-11-09T02:21:37,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741914_1090 (size=12301) 2024-11-09T02:21:37,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118957391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:37,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:37,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:37,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:37,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118957400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:37,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:37,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:37,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:37,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118957410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:37,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:37,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:37,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:37,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T02:21:37,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T02:21:37,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:37,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:37,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:37,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:37,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:37,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:37,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:37,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118957599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:37,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:37,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:37,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:37,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:37,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118957610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:37,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:37,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:37,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:37,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:37,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118957620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:37,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T02:21:37,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:37,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:37,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:37,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:37,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:37,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:37,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/5a7167ecb37e4814944b713932ec3ba1 2024-11-09T02:21:37,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/d59901b62db74e06b04036595755cd62 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/d59901b62db74e06b04036595755cd62 2024-11-09T02:21:37,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T02:21:37,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:37,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:37,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:37,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:37,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:37,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T02:21:37,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:37,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/d59901b62db74e06b04036595755cd62, entries=200, sequenceid=370, filesize=14.4 K 2024-11-09T02:21:37,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/41c577fb7aee4c4f8ce6590cce44b793 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/41c577fb7aee4c4f8ce6590cce44b793 2024-11-09T02:21:37,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/41c577fb7aee4c4f8ce6590cce44b793, entries=150, sequenceid=370, filesize=12.0 K 2024-11-09T02:21:37,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/5a7167ecb37e4814944b713932ec3ba1 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/5a7167ecb37e4814944b713932ec3ba1 2024-11-09T02:21:37,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/5a7167ecb37e4814944b713932ec3ba1, entries=150, sequenceid=370, filesize=12.0 K 2024-11-09T02:21:37,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for aad4fc6835c8f9bd4339c4a54a2c56fe in 648ms, sequenceid=370, compaction requested=true 2024-11-09T02:21:37,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:37,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:21:37,822 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:37,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:37,823 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:37,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:21:37,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:37,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:21:37,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:21:37,824 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:37,824 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/A is initiating minor compaction (all files) 2024-11-09T02:21:37,824 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/A in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:37,824 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:37,824 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/B is initiating minor compaction (all files) 2024-11-09T02:21:37,824 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/283368fe3c10407584d3396e9e041836, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/e6c98b3b2bbb41f89a16a2da15ae5a1b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/d59901b62db74e06b04036595755cd62] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=39.2 K 2024-11-09T02:21:37,824 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/B in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:37,825 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/bec3de834c6542f08a9b46fc1941a407, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/314db20d716f40f1883a21110a5c6191, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/41c577fb7aee4c4f8ce6590cce44b793] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=36.8 K 2024-11-09T02:21:37,825 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting bec3de834c6542f08a9b46fc1941a407, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731118894264 2024-11-09T02:21:37,825 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 283368fe3c10407584d3396e9e041836, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731118894264 2024-11-09T02:21:37,825 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 314db20d716f40f1883a21110a5c6191, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731118894976 2024-11-09T02:21:37,826 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6c98b3b2bbb41f89a16a2da15ae5a1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731118894976 2024-11-09T02:21:37,826 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 41c577fb7aee4c4f8ce6590cce44b793, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731118897141 2024-11-09T02:21:37,827 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting d59901b62db74e06b04036595755cd62, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731118897141 2024-11-09T02:21:37,842 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#B#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:37,843 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/a95e7aa24f3b408a9d20664c39202a4c is 50, key is test_row_0/B:col10/1731118897141/Put/seqid=0 2024-11-09T02:21:37,843 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#A#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:37,844 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/60ee5c26349c454099bcfa668dffa549 is 50, key is test_row_0/A:col10/1731118897141/Put/seqid=0 2024-11-09T02:21:37,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741916_1092 (size=13187) 2024-11-09T02:21:37,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741915_1091 (size=13187) 2024-11-09T02:21:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:37,915 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:21:37,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aad4fc6835c8f9bd4339c4a54a2c56fe 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-09T02:21:37,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/612ee99e0cbe4a1e913c8f53e018d61f is 50, key is test_row_0/A:col10/1731118897270/Put/seqid=0 2024-11-09T02:21:37,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T02:21:37,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:37,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:37,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:37,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:37,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:37,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118957940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:37,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:37,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:37,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118957944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:37,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741917_1093 (size=14741) 2024-11-09T02:21:37,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/612ee99e0cbe4a1e913c8f53e018d61f 2024-11-09T02:21:37,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118957950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:37,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/a620dbb0223c45d0ab2e7453cf52208b is 50, key is test_row_0/B:col10/1731118897270/Put/seqid=0 2024-11-09T02:21:37,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741918_1094 (size=12301) 2024-11-09T02:21:38,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118958060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:38,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:38,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118958060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:38,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:38,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118958060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:38,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T02:21:38,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:38,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:38,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:38,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:38,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:38,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:38,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T02:21:38,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:38,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:38,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:38,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:38,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:38,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:38,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:38,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118958270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:38,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:38,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:38,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:38,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118958270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:38,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:38,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:38,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118958270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:38,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:38,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:38,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:38,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:38,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:38,279 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/60ee5c26349c454099bcfa668dffa549 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/60ee5c26349c454099bcfa668dffa549 2024-11-09T02:21:38,288 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/a95e7aa24f3b408a9d20664c39202a4c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/a95e7aa24f3b408a9d20664c39202a4c 2024-11-09T02:21:38,288 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/A of aad4fc6835c8f9bd4339c4a54a2c56fe into 60ee5c26349c454099bcfa668dffa549(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:38,288 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:38,289 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/A, priority=13, startTime=1731118897822; duration=0sec 2024-11-09T02:21:38,289 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:21:38,289 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:A 2024-11-09T02:21:38,289 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:38,291 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:38,291 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/C is initiating minor compaction (all files) 2024-11-09T02:21:38,291 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/C in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:38,292 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/89a52f0446e742bd8645cf9668d8e7b2, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/5adc6e3199e74cae95e6e52d8c7496f5, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/5a7167ecb37e4814944b713932ec3ba1] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=36.8 K 2024-11-09T02:21:38,293 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89a52f0446e742bd8645cf9668d8e7b2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731118894264 2024-11-09T02:21:38,293 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5adc6e3199e74cae95e6e52d8c7496f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731118894976 2024-11-09T02:21:38,294 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a7167ecb37e4814944b713932ec3ba1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731118897141 2024-11-09T02:21:38,300 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/B of aad4fc6835c8f9bd4339c4a54a2c56fe into a95e7aa24f3b408a9d20664c39202a4c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:38,300 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:38,300 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/B, priority=13, startTime=1731118897822; duration=0sec 2024-11-09T02:21:38,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T02:21:38,300 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:38,300 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:B 2024-11-09T02:21:38,307 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#C#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:38,308 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/743e6ffe702c4ccc921be2366264fdc4 is 50, key is test_row_0/C:col10/1731118897141/Put/seqid=0 2024-11-09T02:21:38,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741919_1095 (size=13187) 2024-11-09T02:21:38,375 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/743e6ffe702c4ccc921be2366264fdc4 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/743e6ffe702c4ccc921be2366264fdc4 2024-11-09T02:21:38,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/a620dbb0223c45d0ab2e7453cf52208b 2024-11-09T02:21:38,387 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/C of aad4fc6835c8f9bd4339c4a54a2c56fe into 743e6ffe702c4ccc921be2366264fdc4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:38,387 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:38,387 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/C, priority=13, startTime=1731118897823; duration=0sec 2024-11-09T02:21:38,389 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:38,389 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:C 2024-11-09T02:21:38,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/d5a85ab1e3604c59aba39756b7c1a44d is 50, key is test_row_0/C:col10/1731118897270/Put/seqid=0 2024-11-09T02:21:38,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T02:21:38,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:38,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:38,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:38,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:38,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:38,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:38,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741920_1096 (size=12301) 2024-11-09T02:21:38,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T02:21:38,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:38,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:38,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:38,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:38,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:38,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:38,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:38,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118958581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:38,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:38,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:38,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:38,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118958583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:38,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:38,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:38,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:38,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118958583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:38,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:38,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:38,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:38,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T02:21:38,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:38,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:38,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:38,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:38,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:38,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/d5a85ab1e3604c59aba39756b7c1a44d 2024-11-09T02:21:38,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/612ee99e0cbe4a1e913c8f53e018d61f as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/612ee99e0cbe4a1e913c8f53e018d61f 2024-11-09T02:21:38,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/612ee99e0cbe4a1e913c8f53e018d61f, entries=200, sequenceid=396, filesize=14.4 K 2024-11-09T02:21:38,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/a620dbb0223c45d0ab2e7453cf52208b as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/a620dbb0223c45d0ab2e7453cf52208b 2024-11-09T02:21:38,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/a620dbb0223c45d0ab2e7453cf52208b, entries=150, sequenceid=396, filesize=12.0 K 2024-11-09T02:21:38,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/d5a85ab1e3604c59aba39756b7c1a44d as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/d5a85ab1e3604c59aba39756b7c1a44d 2024-11-09T02:21:38,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/d5a85ab1e3604c59aba39756b7c1a44d, entries=150, sequenceid=396, filesize=12.0 K 2024-11-09T02:21:38,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for aad4fc6835c8f9bd4339c4a54a2c56fe in 958ms, sequenceid=396, compaction requested=false 2024-11-09T02:21:38,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:38,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T02:21:38,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:38,887 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing aad4fc6835c8f9bd4339c4a54a2c56fe 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-09T02:21:38,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/5b8f59a99ad843f4b7d70a29a052d857 is 50, key is test_row_0/A:col10/1731118897945/Put/seqid=0 2024-11-09T02:21:38,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741921_1097 (size=12301) 2024-11-09T02:21:38,943 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/5b8f59a99ad843f4b7d70a29a052d857 2024-11-09T02:21:38,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/f068df3765d243aa8b0b287613ec1087 is 50, key is test_row_0/B:col10/1731118897945/Put/seqid=0 2024-11-09T02:21:39,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741922_1098 (size=12301) 2024-11-09T02:21:39,019 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/f068df3765d243aa8b0b287613ec1087 2024-11-09T02:21:39,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/cfdfa0aba8114ab5913a3a687a40dd63 is 50, key is test_row_0/C:col10/1731118897945/Put/seqid=0 2024-11-09T02:21:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:39,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:39,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741923_1099 (size=12301) 2024-11-09T02:21:39,099 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/cfdfa0aba8114ab5913a3a687a40dd63 2024-11-09T02:21:39,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/5b8f59a99ad843f4b7d70a29a052d857 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/5b8f59a99ad843f4b7d70a29a052d857 2024-11-09T02:21:39,122 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/5b8f59a99ad843f4b7d70a29a052d857, entries=150, sequenceid=409, filesize=12.0 K 2024-11-09T02:21:39,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/f068df3765d243aa8b0b287613ec1087 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/f068df3765d243aa8b0b287613ec1087 2024-11-09T02:21:39,135 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/f068df3765d243aa8b0b287613ec1087, entries=150, sequenceid=409, filesize=12.0 K 2024-11-09T02:21:39,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/cfdfa0aba8114ab5913a3a687a40dd63 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/cfdfa0aba8114ab5913a3a687a40dd63 2024-11-09T02:21:39,147 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/cfdfa0aba8114ab5913a3a687a40dd63, entries=150, sequenceid=409, filesize=12.0 K 2024-11-09T02:21:39,150 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=100.63 KB/103050 for aad4fc6835c8f9bd4339c4a54a2c56fe in 263ms, sequenceid=409, compaction requested=true 2024-11-09T02:21:39,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:39,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:39,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-09T02:21:39,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-09T02:21:39,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:39,156 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:21:39,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aad4fc6835c8f9bd4339c4a54a2c56fe 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-09T02:21:39,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/cd2983ea29594d4eaa967b57320496d3 is 50, key is test_row_0/A:col10/1731118899147/Put/seqid=0 2024-11-09T02:21:39,175 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-09T02:21:39,176 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9960 sec 2024-11-09T02:21:39,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.0150 sec 2024-11-09T02:21:39,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741924_1100 (size=12301) 2024-11-09T02:21:39,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/cd2983ea29594d4eaa967b57320496d3 2024-11-09T02:21:39,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/1bc067907ae94b018fa85e8c9780b22b is 50, key is test_row_0/B:col10/1731118899147/Put/seqid=0 2024-11-09T02:21:39,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118959240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:39,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118959240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:39,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:39,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:39,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:39,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:39,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:39,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:39,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118959247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:39,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:39,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:39,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:39,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741925_1101 (size=12301) 2024-11-09T02:21:39,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/1bc067907ae94b018fa85e8c9780b22b 2024-11-09T02:21:39,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/6e756ae1cdad49f0b2a7a3fbe89cf4ab is 50, key is test_row_0/C:col10/1731118899147/Put/seqid=0 2024-11-09T02:21:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T02:21:39,310 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:21:39,311 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:21:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-09T02:21:39,313 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:21:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T02:21:39,314 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:21:39,314 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:21:39,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741926_1102 (size=12301) 2024-11-09T02:21:39,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118959360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:39,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118959361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:39,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118959370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:39,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:39,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:39,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:39,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T02:21:39,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-09T02:21:39,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:39,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:39,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:39,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:39,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:39,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118959570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:39,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:39,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118959570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:39,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:39,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:39,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:39,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118959580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:39,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:39,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:39,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:39,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T02:21:39,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-09T02:21:39,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:39,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:39,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:39,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:39,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:39,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:39,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/6e756ae1cdad49f0b2a7a3fbe89cf4ab 2024-11-09T02:21:39,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/cd2983ea29594d4eaa967b57320496d3 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/cd2983ea29594d4eaa967b57320496d3 2024-11-09T02:21:39,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/cd2983ea29594d4eaa967b57320496d3, entries=150, sequenceid=428, filesize=12.0 K 2024-11-09T02:21:39,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/1bc067907ae94b018fa85e8c9780b22b as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/1bc067907ae94b018fa85e8c9780b22b 2024-11-09T02:21:39,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/1bc067907ae94b018fa85e8c9780b22b, entries=150, sequenceid=428, filesize=12.0 K 2024-11-09T02:21:39,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/6e756ae1cdad49f0b2a7a3fbe89cf4ab as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/6e756ae1cdad49f0b2a7a3fbe89cf4ab 2024-11-09T02:21:39,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/6e756ae1cdad49f0b2a7a3fbe89cf4ab, entries=150, sequenceid=428, filesize=12.0 K 2024-11-09T02:21:39,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=107.34 KB/109920 for aad4fc6835c8f9bd4339c4a54a2c56fe in 615ms, sequenceid=428, compaction requested=true 2024-11-09T02:21:39,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:39,771 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T02:21:39,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:21:39,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:39,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:21:39,772 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T02:21:39,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:39,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:21:39,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:21:39,773 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52530 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T02:21:39,774 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/A is initiating minor compaction (all files) 2024-11-09T02:21:39,774 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/A in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:39,774 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/60ee5c26349c454099bcfa668dffa549, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/612ee99e0cbe4a1e913c8f53e018d61f, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/5b8f59a99ad843f4b7d70a29a052d857, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/cd2983ea29594d4eaa967b57320496d3] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=51.3 K 2024-11-09T02:21:39,774 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T02:21:39,775 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/B is initiating minor compaction (all files) 2024-11-09T02:21:39,775 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/B in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:39,775 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/a95e7aa24f3b408a9d20664c39202a4c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/a620dbb0223c45d0ab2e7453cf52208b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/f068df3765d243aa8b0b287613ec1087, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/1bc067907ae94b018fa85e8c9780b22b] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=48.9 K 2024-11-09T02:21:39,775 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 60ee5c26349c454099bcfa668dffa549, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731118897141 2024-11-09T02:21:39,776 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting a95e7aa24f3b408a9d20664c39202a4c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731118897141 2024-11-09T02:21:39,776 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 612ee99e0cbe4a1e913c8f53e018d61f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731118897267 2024-11-09T02:21:39,776 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting a620dbb0223c45d0ab2e7453cf52208b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731118897267 2024-11-09T02:21:39,776 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting f068df3765d243aa8b0b287613ec1087, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731118897933 2024-11-09T02:21:39,777 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b8f59a99ad843f4b7d70a29a052d857, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731118897933 2024-11-09T02:21:39,777 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bc067907ae94b018fa85e8c9780b22b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731118899099 2024-11-09T02:21:39,777 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting cd2983ea29594d4eaa967b57320496d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731118899099 2024-11-09T02:21:39,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-09T02:21:39,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:39,789 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing aad4fc6835c8f9bd4339c4a54a2c56fe 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-09T02:21:39,802 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#B#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:39,803 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/aea1c0ed24144000932a634815212e69 is 50, key is test_row_0/B:col10/1731118899147/Put/seqid=0 2024-11-09T02:21:39,810 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#A#compaction#91 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:39,811 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/d374ead9ad3645978504331ee7733150 is 50, key is test_row_0/A:col10/1731118899147/Put/seqid=0 2024-11-09T02:21:39,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/a5aaa4f8eaed4f83986164e24becf02c is 50, key is test_row_0/A:col10/1731118899238/Put/seqid=0 2024-11-09T02:21:39,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741927_1103 (size=13323) 2024-11-09T02:21:39,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:39,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741928_1104 (size=13323) 2024-11-09T02:21:39,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741929_1105 (size=12301) 2024-11-09T02:21:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T02:21:39,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118959933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:39,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:39,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118959943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118959945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:40,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118960050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:40,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:40,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:40,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118960090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:40,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:40,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:40,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:40,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118960090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:40,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:40,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:40,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:40,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:40,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118960270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:40,294 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/aea1c0ed24144000932a634815212e69 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/aea1c0ed24144000932a634815212e69 2024-11-09T02:21:40,302 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/B of aad4fc6835c8f9bd4339c4a54a2c56fe into aea1c0ed24144000932a634815212e69(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:40,302 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:40,302 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/B, priority=12, startTime=1731118899772; duration=0sec 2024-11-09T02:21:40,303 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:21:40,303 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:B 2024-11-09T02:21:40,304 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T02:21:40,307 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T02:21:40,307 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/C is initiating minor compaction (all files) 2024-11-09T02:21:40,307 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/C in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:40,308 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/743e6ffe702c4ccc921be2366264fdc4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/d5a85ab1e3604c59aba39756b7c1a44d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/cfdfa0aba8114ab5913a3a687a40dd63, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/6e756ae1cdad49f0b2a7a3fbe89cf4ab] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=48.9 K 2024-11-09T02:21:40,309 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 743e6ffe702c4ccc921be2366264fdc4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731118897141 2024-11-09T02:21:40,310 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5a85ab1e3604c59aba39756b7c1a44d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731118897267 2024-11-09T02:21:40,311 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfdfa0aba8114ab5913a3a687a40dd63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731118897933 2024-11-09T02:21:40,311 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/d374ead9ad3645978504331ee7733150 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/d374ead9ad3645978504331ee7733150 2024-11-09T02:21:40,311 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e756ae1cdad49f0b2a7a3fbe89cf4ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731118899099 2024-11-09T02:21:40,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:40,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118960300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:40,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:40,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:40,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:40,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118960311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:40,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:40,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:40,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:40,339 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/a5aaa4f8eaed4f83986164e24becf02c 2024-11-09T02:21:40,348 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/A of aad4fc6835c8f9bd4339c4a54a2c56fe into d374ead9ad3645978504331ee7733150(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:40,349 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:40,349 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/A, priority=12, startTime=1731118899771; duration=0sec 2024-11-09T02:21:40,349 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:40,349 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:A 2024-11-09T02:21:40,377 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#C#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:40,378 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/d681d4f74ec74c44a9bb97c59f1d0aa9 is 50, key is test_row_0/C:col10/1731118899147/Put/seqid=0 2024-11-09T02:21:40,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/1c34de80c4b24ca682b9f2c7adeb2b73 is 50, key is test_row_0/B:col10/1731118899238/Put/seqid=0 2024-11-09T02:21:40,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T02:21:40,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741930_1106 (size=13323) 2024-11-09T02:21:40,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741931_1107 (size=12301) 2024-11-09T02:21:40,489 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/d681d4f74ec74c44a9bb97c59f1d0aa9 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/d681d4f74ec74c44a9bb97c59f1d0aa9 2024-11-09T02:21:40,515 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/C of aad4fc6835c8f9bd4339c4a54a2c56fe into d681d4f74ec74c44a9bb97c59f1d0aa9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:40,515 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:40,515 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/C, priority=12, startTime=1731118899772; duration=0sec 2024-11-09T02:21:40,515 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:40,515 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:C 2024-11-09T02:21:40,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:40,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118960580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:40,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:40,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:40,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:40,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:40,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118960630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:40,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:40,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:40,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:40,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:40,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118960641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:40,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:40,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:40,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:40,858 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/1c34de80c4b24ca682b9f2c7adeb2b73 2024-11-09T02:21:40,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/6cbdfcd187394e26853ec4993d78f3f9 is 50, key is test_row_0/C:col10/1731118899238/Put/seqid=0 2024-11-09T02:21:40,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741932_1108 (size=12301) 2024-11-09T02:21:40,903 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/6cbdfcd187394e26853ec4993d78f3f9 2024-11-09T02:21:40,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/a5aaa4f8eaed4f83986164e24becf02c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/a5aaa4f8eaed4f83986164e24becf02c 2024-11-09T02:21:40,923 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/a5aaa4f8eaed4f83986164e24becf02c, entries=150, sequenceid=447, filesize=12.0 K 2024-11-09T02:21:40,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/1c34de80c4b24ca682b9f2c7adeb2b73 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/1c34de80c4b24ca682b9f2c7adeb2b73 2024-11-09T02:21:40,934 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/1c34de80c4b24ca682b9f2c7adeb2b73, entries=150, sequenceid=447, filesize=12.0 K 2024-11-09T02:21:40,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/6cbdfcd187394e26853ec4993d78f3f9 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/6cbdfcd187394e26853ec4993d78f3f9 2024-11-09T02:21:40,954 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/6cbdfcd187394e26853ec4993d78f3f9, entries=150, sequenceid=447, filesize=12.0 K 2024-11-09T02:21:40,956 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for aad4fc6835c8f9bd4339c4a54a2c56fe in 1167ms, sequenceid=447, compaction requested=false 2024-11-09T02:21:40,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:40,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:40,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-09T02:21:40,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-09T02:21:40,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-09T02:21:40,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6510 sec 2024-11-09T02:21:40,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.6630 sec 2024-11-09T02:21:41,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:41,093 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:21:41,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aad4fc6835c8f9bd4339c4a54a2c56fe 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-09T02:21:41,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/77f1e82d0025482998e2827591f545db is 50, key is test_row_0/A:col10/1731118901092/Put/seqid=0 2024-11-09T02:21:41,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741933_1109 (size=14741) 2024-11-09T02:21:41,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/77f1e82d0025482998e2827591f545db 2024-11-09T02:21:41,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/dfe562a962f64b019bf49d408f985e89 is 50, key is test_row_0/B:col10/1731118901092/Put/seqid=0 2024-11-09T02:21:41,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741934_1110 (size=12301) 2024-11-09T02:21:41,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:41,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118961173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:41,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:41,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:41,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:41,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:41,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118961182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:41,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:41,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:41,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:41,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:41,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118961182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:41,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:41,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:41,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:41,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:41,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118961289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:41,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:41,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:41,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118961290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:41,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:41,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:41,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:41,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:41,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:41,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:41,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118961300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:41,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:41,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:41,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T02:21:41,460 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:21:41,461 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:21:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-09T02:21:41,465 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:21:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T02:21:41,466 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:21:41,466 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:21:41,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118961510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:41,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:41,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:41,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:41,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118961511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:41,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:41,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:41,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:41,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118961513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:41,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:41,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:41,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:41,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/dfe562a962f64b019bf49d408f985e89 2024-11-09T02:21:41,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T02:21:41,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/ce83d450b692454ba9a0cc54d3c8fe0a is 50, key is test_row_0/C:col10/1731118901092/Put/seqid=0 2024-11-09T02:21:41,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T02:21:41,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:41,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:41,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:41,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:41,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:41,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:41,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741935_1111 (size=12301) 2024-11-09T02:21:41,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T02:21:41,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:41,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:41,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:41,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:41,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:41,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T02:21:41,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:41,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:41,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58394 deadline: 1731118961820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:41,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58372 deadline: 1731118961820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:41,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:41,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58368 deadline: 1731118961820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:41,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:41,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:41,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:41,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:41,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:41,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:41,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:41,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:21:41,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T02:21:41,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:41,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:41,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:41,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:41,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:41,985 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-09T02:21:41,985 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-09T02:21:41,985 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:21:41,985 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:21:41,986 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-09T02:21:41,986 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:21:41,988 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-09T02:21:41,988 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:21:41,991 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:41,991 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:41,991 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:41,991 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:41,991 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:41,991 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:41,991 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:21:41,991 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:41,992 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:41,992 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:21:41,991 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:21:41,992 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:21:42,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/ce83d450b692454ba9a0cc54d3c8fe0a 2024-11-09T02:21:42,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/77f1e82d0025482998e2827591f545db as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/77f1e82d0025482998e2827591f545db 2024-11-09T02:21:42,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/77f1e82d0025482998e2827591f545db, entries=200, sequenceid=468, filesize=14.4 K 2024-11-09T02:21:42,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/dfe562a962f64b019bf49d408f985e89 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/dfe562a962f64b019bf49d408f985e89 2024-11-09T02:21:42,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/dfe562a962f64b019bf49d408f985e89, entries=150, sequenceid=468, filesize=12.0 K 2024-11-09T02:21:42,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/ce83d450b692454ba9a0cc54d3c8fe0a as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/ce83d450b692454ba9a0cc54d3c8fe0a 2024-11-09T02:21:42,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/ce83d450b692454ba9a0cc54d3c8fe0a, entries=150, sequenceid=468, filesize=12.0 K 2024-11-09T02:21:42,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for aad4fc6835c8f9bd4339c4a54a2c56fe in 957ms, sequenceid=468, compaction requested=true 2024-11-09T02:21:42,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:42,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:21:42,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:42,050 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:42,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:21:42,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:42,050 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:42,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aad4fc6835c8f9bd4339c4a54a2c56fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:21:42,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:21:42,051 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:42,051 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:42,052 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/B is initiating minor compaction (all files) 2024-11-09T02:21:42,052 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/A is initiating minor compaction (all files) 2024-11-09T02:21:42,052 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/B in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:42,052 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/A in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:42,052 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/aea1c0ed24144000932a634815212e69, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/1c34de80c4b24ca682b9f2c7adeb2b73, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/dfe562a962f64b019bf49d408f985e89] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=37.0 K 2024-11-09T02:21:42,052 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/d374ead9ad3645978504331ee7733150, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/a5aaa4f8eaed4f83986164e24becf02c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/77f1e82d0025482998e2827591f545db] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=39.4 K 2024-11-09T02:21:42,052 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting aea1c0ed24144000932a634815212e69, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731118899099 2024-11-09T02:21:42,052 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting d374ead9ad3645978504331ee7733150, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731118899099 2024-11-09T02:21:42,052 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c34de80c4b24ca682b9f2c7adeb2b73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731118899208 2024-11-09T02:21:42,053 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5aaa4f8eaed4f83986164e24becf02c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731118899208 2024-11-09T02:21:42,053 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting dfe562a962f64b019bf49d408f985e89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731118899911 2024-11-09T02:21:42,053 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77f1e82d0025482998e2827591f545db, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731118899908 2024-11-09T02:21:42,061 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#B#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:42,062 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/a4524aa67f464ffeaf5e8264c88840df is 50, key is test_row_0/B:col10/1731118901092/Put/seqid=0 2024-11-09T02:21:42,066 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#A#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:42,066 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/b211d95fcc4f487e9b823edd7fc3a36f is 50, key is test_row_0/A:col10/1731118901092/Put/seqid=0 2024-11-09T02:21:42,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741936_1112 (size=13425) 2024-11-09T02:21:42,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741937_1113 (size=13425) 2024-11-09T02:21:42,080 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/b211d95fcc4f487e9b823edd7fc3a36f as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/b211d95fcc4f487e9b823edd7fc3a36f 2024-11-09T02:21:42,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T02:21:42,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:42,085 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing aad4fc6835c8f9bd4339c4a54a2c56fe 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-09T02:21:42,086 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/A of aad4fc6835c8f9bd4339c4a54a2c56fe into b211d95fcc4f487e9b823edd7fc3a36f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:42,086 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:42,086 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/A, priority=13, startTime=1731118902050; duration=0sec 2024-11-09T02:21:42,086 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:21:42,086 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:A 2024-11-09T02:21:42,087 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:42,088 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:42,088 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): aad4fc6835c8f9bd4339c4a54a2c56fe/C is initiating minor compaction (all files) 2024-11-09T02:21:42,088 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aad4fc6835c8f9bd4339c4a54a2c56fe/C in TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:42,088 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/d681d4f74ec74c44a9bb97c59f1d0aa9, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/6cbdfcd187394e26853ec4993d78f3f9, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/ce83d450b692454ba9a0cc54d3c8fe0a] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp, totalSize=37.0 K 2024-11-09T02:21:42,089 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting d681d4f74ec74c44a9bb97c59f1d0aa9, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731118899099 2024-11-09T02:21:42,089 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cbdfcd187394e26853ec4993d78f3f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731118899208 2024-11-09T02:21:42,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T02:21:42,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/fb44a7c3f9b34af29cb40e7c985baa94 is 50, key is test_row_0/A:col10/1731118901177/Put/seqid=0 2024-11-09T02:21:42,092 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce83d450b692454ba9a0cc54d3c8fe0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731118899911 2024-11-09T02:21:42,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741938_1114 (size=12301) 2024-11-09T02:21:42,103 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aad4fc6835c8f9bd4339c4a54a2c56fe#C#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:42,103 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/0375e862fe5946b9b4b5bd742436aa99 is 50, key is test_row_0/C:col10/1731118901092/Put/seqid=0 2024-11-09T02:21:42,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741939_1115 (size=13425) 2024-11-09T02:21:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:42,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. as already flushing 2024-11-09T02:21:42,332 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-09T02:21:42,332 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-09T02:21:42,332 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-09T02:21:42,333 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:21:42,333 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:21:42,333 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:21:42,333 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:42,333 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:42,333 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:42,333 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:42,333 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:42,333 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:42,333 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:21:42,333 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:21:42,333 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:21:42,476 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/a4524aa67f464ffeaf5e8264c88840df as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/a4524aa67f464ffeaf5e8264c88840df 2024-11-09T02:21:42,483 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/B of aad4fc6835c8f9bd4339c4a54a2c56fe into a4524aa67f464ffeaf5e8264c88840df(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:42,483 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:42,483 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/B, priority=13, startTime=1731118902050; duration=0sec 2024-11-09T02:21:42,483 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:42,483 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:B 2024-11-09T02:21:42,497 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/fb44a7c3f9b34af29cb40e7c985baa94 2024-11-09T02:21:42,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/f03f6d2daf5949a7b8b0e17f636c2d20 is 50, key is test_row_0/B:col10/1731118901177/Put/seqid=0 2024-11-09T02:21:42,518 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/0375e862fe5946b9b4b5bd742436aa99 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/0375e862fe5946b9b4b5bd742436aa99 2024-11-09T02:21:42,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741940_1116 (size=12301) 2024-11-09T02:21:42,526 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aad4fc6835c8f9bd4339c4a54a2c56fe/C of aad4fc6835c8f9bd4339c4a54a2c56fe into 0375e862fe5946b9b4b5bd742436aa99(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:42,526 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:42,526 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., storeName=aad4fc6835c8f9bd4339c4a54a2c56fe/C, priority=13, startTime=1731118902050; duration=0sec 2024-11-09T02:21:42,526 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:42,526 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aad4fc6835c8f9bd4339c4a54a2c56fe:C 2024-11-09T02:21:42,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T02:21:42,927 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/f03f6d2daf5949a7b8b0e17f636c2d20 2024-11-09T02:21:42,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/b39853b4a0b94241bc7609a4fdb70847 is 50, key is test_row_0/C:col10/1731118901177/Put/seqid=0 2024-11-09T02:21:42,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741941_1117 (size=12301) 2024-11-09T02:21:42,946 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/b39853b4a0b94241bc7609a4fdb70847 2024-11-09T02:21:42,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/fb44a7c3f9b34af29cb40e7c985baa94 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/fb44a7c3f9b34af29cb40e7c985baa94 2024-11-09T02:21:42,967 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/fb44a7c3f9b34af29cb40e7c985baa94, entries=150, sequenceid=488, filesize=12.0 K 2024-11-09T02:21:42,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/f03f6d2daf5949a7b8b0e17f636c2d20 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/f03f6d2daf5949a7b8b0e17f636c2d20 2024-11-09T02:21:42,976 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/f03f6d2daf5949a7b8b0e17f636c2d20, entries=150, sequenceid=488, filesize=12.0 K 2024-11-09T02:21:42,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/b39853b4a0b94241bc7609a4fdb70847 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/b39853b4a0b94241bc7609a4fdb70847 2024-11-09T02:21:42,982 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/b39853b4a0b94241bc7609a4fdb70847, entries=150, sequenceid=488, filesize=12.0 K 2024-11-09T02:21:42,983 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=20.13 KB/20610 for aad4fc6835c8f9bd4339c4a54a2c56fe in 898ms, sequenceid=488, compaction requested=false 2024-11-09T02:21:42,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for aad4fc6835c8f9bd4339c4a54a2c56fe: 2024-11-09T02:21:42,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:42,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-09T02:21:42,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-09T02:21:42,990 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-09T02:21:42,991 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5200 sec 2024-11-09T02:21:42,992 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.5300 sec 2024-11-09T02:21:43,498 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T02:21:43,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T02:21:43,610 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:21:45,871 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-09T02:21:45,872 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:21:45,872 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:45,872 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:45,872 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:21:45,931 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-09T02:21:45,931 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:21:45,931 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:45,931 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:45,931 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:21:45,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T02:21:45,932 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-09T02:21:45,932 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-11-09T02:21:45,932 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 123 2024-11-09T02:21:45,932 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-09T02:21:45,932 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-09T02:21:45,932 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T02:21:45,932 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2294 2024-11-09T02:21:45,932 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2241 2024-11-09T02:21:45,932 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T02:21:45,932 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1104 2024-11-09T02:21:45,932 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3308 rows 2024-11-09T02:21:45,932 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1104 2024-11-09T02:21:45,932 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3311 rows 2024-11-09T02:21:45,932 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T02:21:45,932 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:21:45,932 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:45,933 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:45,933 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:21:45,936 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T02:21:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T02:21:45,943 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118905943"}]},"ts":"1731118905943"} 2024-11-09T02:21:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T02:21:45,945 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T02:21:45,945 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T02:21:45,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T02:21:45,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aad4fc6835c8f9bd4339c4a54a2c56fe, UNASSIGN}] 2024-11-09T02:21:45,952 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aad4fc6835c8f9bd4339c4a54a2c56fe, UNASSIGN 2024-11-09T02:21:45,953 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=aad4fc6835c8f9bd4339c4a54a2c56fe, regionState=CLOSING, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:45,956 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aad4fc6835c8f9bd4339c4a54a2c56fe, UNASSIGN because future has completed 2024-11-09T02:21:45,956 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T02:21:45,957 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223}] 2024-11-09T02:21:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T02:21:46,115 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:46,115 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T02:21:46,116 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing aad4fc6835c8f9bd4339c4a54a2c56fe, disabling compactions & flushes 2024-11-09T02:21:46,116 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:46,116 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:46,116 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. after waiting 0 ms 2024-11-09T02:21:46,116 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:46,116 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing aad4fc6835c8f9bd4339c4a54a2c56fe 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-09T02:21:46,122 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/ab76d86317534979b74a53b0054edfce is 50, key is test_row_0/A:col10/1731118905930/Put/seqid=0 2024-11-09T02:21:46,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741942_1118 (size=9857) 2024-11-09T02:21:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T02:21:46,543 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/ab76d86317534979b74a53b0054edfce 2024-11-09T02:21:46,551 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/5ffac1f0926944db815362b8aabbcd4a is 50, key is test_row_0/B:col10/1731118905930/Put/seqid=0 2024-11-09T02:21:46,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741943_1119 (size=9857) 2024-11-09T02:21:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T02:21:46,955 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/5ffac1f0926944db815362b8aabbcd4a 2024-11-09T02:21:46,963 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/2c7170234dd74304933ba00989c9b5e6 is 50, key is test_row_0/C:col10/1731118905930/Put/seqid=0 2024-11-09T02:21:46,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741944_1120 (size=9857) 2024-11-09T02:21:47,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T02:21:47,373 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/2c7170234dd74304933ba00989c9b5e6 2024-11-09T02:21:47,380 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/A/ab76d86317534979b74a53b0054edfce as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/ab76d86317534979b74a53b0054edfce 2024-11-09T02:21:47,386 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/ab76d86317534979b74a53b0054edfce, entries=100, sequenceid=498, filesize=9.6 K 2024-11-09T02:21:47,387 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/B/5ffac1f0926944db815362b8aabbcd4a as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/5ffac1f0926944db815362b8aabbcd4a 2024-11-09T02:21:47,392 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/5ffac1f0926944db815362b8aabbcd4a, entries=100, sequenceid=498, filesize=9.6 K 2024-11-09T02:21:47,393 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/.tmp/C/2c7170234dd74304933ba00989c9b5e6 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/2c7170234dd74304933ba00989c9b5e6 2024-11-09T02:21:47,398 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/2c7170234dd74304933ba00989c9b5e6, entries=100, sequenceid=498, filesize=9.6 K 2024-11-09T02:21:47,399 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for aad4fc6835c8f9bd4339c4a54a2c56fe in 1283ms, sequenceid=498, compaction requested=true 2024-11-09T02:21:47,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/67b4068fcca6421587a38bbbe88e00eb, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/28ef4f78a24b4ebe87ca4c5351924545, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/660e55d3404c41f1a6eb68490419a7c9, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/e825b14668f84815adf9f3737cb83bbd, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/21e4b9250c474c9eb9aa69554d8e062a, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/f17353b2b62e4a609243a507e190d10d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/1e4e9ef8b33b4d4a87af28a66b082ad1, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/fe53240e16d34361b591037a42d0665a, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/18d355d928e046a4911456e32750797c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/b8a7bf1cd8004ddea168819cb1c1bcb2, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/0c98da9fc29f474787e8d994dd30f67e, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/4ec577bbd5274abe86be24b3b923dbf1, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/4378d5ddbfd64047b5d3af53f7212be1, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/092ce4f874f04e728bedda5f9656505f, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/01d693c74af34faeb1f02f011e6174f9, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/284283e9735c443d91a55e5ef3c3a9c3, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/71b51448009b4a9a92a0a7714c4534cf, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/c5dd0d14315d4ae490071b84cc0e2e3a, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/17a6430a2ced4021a62521b95e94c7d5, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/cd794c0ee6e84a20a4ac6cd3934ffec3, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/2196ca96aa04481ea11b5d125f8165cd, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/c8433e308bd84b84a26415ae77ccd7f3, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/f3b1e0cf7d7f4e1ab73d5171d8b6a11d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/283368fe3c10407584d3396e9e041836, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/e6c98b3b2bbb41f89a16a2da15ae5a1b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/d59901b62db74e06b04036595755cd62, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/60ee5c26349c454099bcfa668dffa549, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/612ee99e0cbe4a1e913c8f53e018d61f, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/5b8f59a99ad843f4b7d70a29a052d857, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/d374ead9ad3645978504331ee7733150, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/cd2983ea29594d4eaa967b57320496d3, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/a5aaa4f8eaed4f83986164e24becf02c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/77f1e82d0025482998e2827591f545db] to archive 2024-11-09T02:21:47,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T02:21:47,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/67b4068fcca6421587a38bbbe88e00eb to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/67b4068fcca6421587a38bbbe88e00eb 2024-11-09T02:21:47,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/28ef4f78a24b4ebe87ca4c5351924545 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/28ef4f78a24b4ebe87ca4c5351924545 2024-11-09T02:21:47,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/660e55d3404c41f1a6eb68490419a7c9 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/660e55d3404c41f1a6eb68490419a7c9 2024-11-09T02:21:47,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/e825b14668f84815adf9f3737cb83bbd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/e825b14668f84815adf9f3737cb83bbd 2024-11-09T02:21:47,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/21e4b9250c474c9eb9aa69554d8e062a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/21e4b9250c474c9eb9aa69554d8e062a 2024-11-09T02:21:47,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/f17353b2b62e4a609243a507e190d10d to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/f17353b2b62e4a609243a507e190d10d 2024-11-09T02:21:47,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/1e4e9ef8b33b4d4a87af28a66b082ad1 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/1e4e9ef8b33b4d4a87af28a66b082ad1 2024-11-09T02:21:47,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/fe53240e16d34361b591037a42d0665a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/fe53240e16d34361b591037a42d0665a 2024-11-09T02:21:47,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/18d355d928e046a4911456e32750797c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/18d355d928e046a4911456e32750797c 2024-11-09T02:21:47,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/b8a7bf1cd8004ddea168819cb1c1bcb2 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/b8a7bf1cd8004ddea168819cb1c1bcb2 2024-11-09T02:21:47,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/0c98da9fc29f474787e8d994dd30f67e to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/0c98da9fc29f474787e8d994dd30f67e 2024-11-09T02:21:47,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/4ec577bbd5274abe86be24b3b923dbf1 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/4ec577bbd5274abe86be24b3b923dbf1 2024-11-09T02:21:47,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/4378d5ddbfd64047b5d3af53f7212be1 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/4378d5ddbfd64047b5d3af53f7212be1 2024-11-09T02:21:47,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/092ce4f874f04e728bedda5f9656505f to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/092ce4f874f04e728bedda5f9656505f 2024-11-09T02:21:47,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/01d693c74af34faeb1f02f011e6174f9 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/01d693c74af34faeb1f02f011e6174f9 2024-11-09T02:21:47,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/284283e9735c443d91a55e5ef3c3a9c3 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/284283e9735c443d91a55e5ef3c3a9c3 2024-11-09T02:21:47,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/71b51448009b4a9a92a0a7714c4534cf to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/71b51448009b4a9a92a0a7714c4534cf 2024-11-09T02:21:47,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/c5dd0d14315d4ae490071b84cc0e2e3a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/c5dd0d14315d4ae490071b84cc0e2e3a 2024-11-09T02:21:47,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/17a6430a2ced4021a62521b95e94c7d5 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/17a6430a2ced4021a62521b95e94c7d5 2024-11-09T02:21:47,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/cd794c0ee6e84a20a4ac6cd3934ffec3 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/cd794c0ee6e84a20a4ac6cd3934ffec3 2024-11-09T02:21:47,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/2196ca96aa04481ea11b5d125f8165cd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/2196ca96aa04481ea11b5d125f8165cd 2024-11-09T02:21:47,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/c8433e308bd84b84a26415ae77ccd7f3 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/c8433e308bd84b84a26415ae77ccd7f3 2024-11-09T02:21:47,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/f3b1e0cf7d7f4e1ab73d5171d8b6a11d to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/f3b1e0cf7d7f4e1ab73d5171d8b6a11d 2024-11-09T02:21:47,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/283368fe3c10407584d3396e9e041836 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/283368fe3c10407584d3396e9e041836 2024-11-09T02:21:47,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/e6c98b3b2bbb41f89a16a2da15ae5a1b to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/e6c98b3b2bbb41f89a16a2da15ae5a1b 2024-11-09T02:21:47,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/d59901b62db74e06b04036595755cd62 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/d59901b62db74e06b04036595755cd62 2024-11-09T02:21:47,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/60ee5c26349c454099bcfa668dffa549 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/60ee5c26349c454099bcfa668dffa549 2024-11-09T02:21:47,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/612ee99e0cbe4a1e913c8f53e018d61f to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/612ee99e0cbe4a1e913c8f53e018d61f 2024-11-09T02:21:47,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/5b8f59a99ad843f4b7d70a29a052d857 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/5b8f59a99ad843f4b7d70a29a052d857 2024-11-09T02:21:47,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/d374ead9ad3645978504331ee7733150 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/d374ead9ad3645978504331ee7733150 2024-11-09T02:21:47,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/cd2983ea29594d4eaa967b57320496d3 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/cd2983ea29594d4eaa967b57320496d3 2024-11-09T02:21:47,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/a5aaa4f8eaed4f83986164e24becf02c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/a5aaa4f8eaed4f83986164e24becf02c 2024-11-09T02:21:47,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/77f1e82d0025482998e2827591f545db to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/77f1e82d0025482998e2827591f545db 2024-11-09T02:21:47,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/f84d955ed2ce40b69754de6a9b2b3cc8, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/3a7bfe55802d481589145ee90edb6ce4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/28f2d0655ce1487abf8e5d67a15d63de, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/cffa16b74f224f64be6057b0a3a95075, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/453a05298c4f474594353bbd05fab6f8, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/797df0795dee49bbb6c29e64878c0051, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/50184eeeab03498a908f90209353d774, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/ef50fa4d6cb0480f9c9feda6fe803e24, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/fd789be3533c4899852d7d5988f16bd9, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/6ceb74479a71458ab34dcb3f48a5d2af, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/900e2f3e629040779397f1c43cbaf83c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/d9a3eb0a2bfe451a94a7028ee8386258, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/819df7ff05414a02bb91325ceeff0ef6, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/6d31eca5654d44c1ade940a8bdd36d82, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/45e8b30696f6439db032a6f1593d3039, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/ed8e341a01bb42bea2a3ba2526869d2c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/bd33c2a195214cd19723c68a22560d06, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/14fbc653f16c43808085ff4004575b02, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/9e4c557ec0b645d5952ee31c23d44f30, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/3138a97029374249a4b8e4b077a810da, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/11f3233a04894e11be54517282c479ea, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/4a822b37bf9d43d0816c0e73817fa304, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/bec3de834c6542f08a9b46fc1941a407, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/1ecb1b4676284af8b57ffe2c157e822c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/314db20d716f40f1883a21110a5c6191, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/a95e7aa24f3b408a9d20664c39202a4c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/41c577fb7aee4c4f8ce6590cce44b793, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/a620dbb0223c45d0ab2e7453cf52208b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/f068df3765d243aa8b0b287613ec1087, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/aea1c0ed24144000932a634815212e69, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/1bc067907ae94b018fa85e8c9780b22b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/1c34de80c4b24ca682b9f2c7adeb2b73, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/dfe562a962f64b019bf49d408f985e89] to archive 2024-11-09T02:21:47,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T02:21:47,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/f84d955ed2ce40b69754de6a9b2b3cc8 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/f84d955ed2ce40b69754de6a9b2b3cc8 2024-11-09T02:21:47,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/3a7bfe55802d481589145ee90edb6ce4 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/3a7bfe55802d481589145ee90edb6ce4 2024-11-09T02:21:47,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/28f2d0655ce1487abf8e5d67a15d63de to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/28f2d0655ce1487abf8e5d67a15d63de 2024-11-09T02:21:47,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/cffa16b74f224f64be6057b0a3a95075 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/cffa16b74f224f64be6057b0a3a95075 2024-11-09T02:21:47,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/453a05298c4f474594353bbd05fab6f8 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/453a05298c4f474594353bbd05fab6f8 2024-11-09T02:21:47,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/797df0795dee49bbb6c29e64878c0051 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/797df0795dee49bbb6c29e64878c0051 2024-11-09T02:21:47,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/50184eeeab03498a908f90209353d774 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/50184eeeab03498a908f90209353d774 2024-11-09T02:21:47,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/ef50fa4d6cb0480f9c9feda6fe803e24 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/ef50fa4d6cb0480f9c9feda6fe803e24 2024-11-09T02:21:47,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/fd789be3533c4899852d7d5988f16bd9 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/fd789be3533c4899852d7d5988f16bd9 2024-11-09T02:21:47,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/6ceb74479a71458ab34dcb3f48a5d2af to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/6ceb74479a71458ab34dcb3f48a5d2af 2024-11-09T02:21:47,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/900e2f3e629040779397f1c43cbaf83c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/900e2f3e629040779397f1c43cbaf83c 2024-11-09T02:21:47,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/d9a3eb0a2bfe451a94a7028ee8386258 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/d9a3eb0a2bfe451a94a7028ee8386258 2024-11-09T02:21:47,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/819df7ff05414a02bb91325ceeff0ef6 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/819df7ff05414a02bb91325ceeff0ef6 2024-11-09T02:21:47,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/6d31eca5654d44c1ade940a8bdd36d82 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/6d31eca5654d44c1ade940a8bdd36d82 2024-11-09T02:21:47,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/45e8b30696f6439db032a6f1593d3039 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/45e8b30696f6439db032a6f1593d3039 2024-11-09T02:21:47,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/ed8e341a01bb42bea2a3ba2526869d2c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/ed8e341a01bb42bea2a3ba2526869d2c 2024-11-09T02:21:47,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/bd33c2a195214cd19723c68a22560d06 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/bd33c2a195214cd19723c68a22560d06 2024-11-09T02:21:47,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/14fbc653f16c43808085ff4004575b02 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/14fbc653f16c43808085ff4004575b02 2024-11-09T02:21:47,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/9e4c557ec0b645d5952ee31c23d44f30 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/9e4c557ec0b645d5952ee31c23d44f30 2024-11-09T02:21:47,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/3138a97029374249a4b8e4b077a810da to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/3138a97029374249a4b8e4b077a810da 2024-11-09T02:21:47,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/11f3233a04894e11be54517282c479ea to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/11f3233a04894e11be54517282c479ea 2024-11-09T02:21:47,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/4a822b37bf9d43d0816c0e73817fa304 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/4a822b37bf9d43d0816c0e73817fa304 2024-11-09T02:21:47,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/bec3de834c6542f08a9b46fc1941a407 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/bec3de834c6542f08a9b46fc1941a407 2024-11-09T02:21:47,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/1ecb1b4676284af8b57ffe2c157e822c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/1ecb1b4676284af8b57ffe2c157e822c 2024-11-09T02:21:47,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/314db20d716f40f1883a21110a5c6191 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/314db20d716f40f1883a21110a5c6191 2024-11-09T02:21:47,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/a95e7aa24f3b408a9d20664c39202a4c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/a95e7aa24f3b408a9d20664c39202a4c 2024-11-09T02:21:47,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/41c577fb7aee4c4f8ce6590cce44b793 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/41c577fb7aee4c4f8ce6590cce44b793 2024-11-09T02:21:47,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/a620dbb0223c45d0ab2e7453cf52208b to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/a620dbb0223c45d0ab2e7453cf52208b 2024-11-09T02:21:47,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/f068df3765d243aa8b0b287613ec1087 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/f068df3765d243aa8b0b287613ec1087 2024-11-09T02:21:47,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/aea1c0ed24144000932a634815212e69 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/aea1c0ed24144000932a634815212e69 2024-11-09T02:21:47,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/1bc067907ae94b018fa85e8c9780b22b to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/1bc067907ae94b018fa85e8c9780b22b 2024-11-09T02:21:47,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/1c34de80c4b24ca682b9f2c7adeb2b73 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/1c34de80c4b24ca682b9f2c7adeb2b73 2024-11-09T02:21:47,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/dfe562a962f64b019bf49d408f985e89 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/dfe562a962f64b019bf49d408f985e89 2024-11-09T02:21:47,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/f6475ea7cf934960868b5fa942b61524, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/59a3c3e9b11e43d19e54750a4293caa8, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/aa09c79478cb4179b4fe6ac78b01099d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/1159651d53b4478d800acf8b8e595c57, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/a17e61f611454578a4272576a50c13c4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/591bdcac8caf45e3b7b8718cbfbb7470, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/dc3615df843a47e893aad8665efa9e79, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/1aae7a0b1a5646dc9e463b8ae2db7978, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/efa69af4df7743a1a2f4bc5a2ea59d5b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/915922a86d1942b2bdcdfca812683c8b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/85f2d4a2f325444faae15a5f06f2e3aa, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/ab625d01979d425fa63b0da3886ecf06, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/16873e7b44f64cd5850a010084a41617, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/b513b3bb91ec4dbc99c32afe637f9546, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/06875fe74110448785b4cafa557261c9, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/a20582a74fe941838615457545d0e9cc, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/389b932d05e1435086771861cbba5ab8, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/8ef68d74652847d9a30e9679a1ae38e3, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/fb2cc24c5c2c4207afe9e07ee19db76b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/e28835f448e3485c953ba2ad78a40e91, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/08c51ce536a64d578e81ee34c6e59ee7, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/ad024116184d413fa92479f04c165073, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/89a52f0446e742bd8645cf9668d8e7b2, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/59456abaa30e443c922e135ed06dfc15, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/5adc6e3199e74cae95e6e52d8c7496f5, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/743e6ffe702c4ccc921be2366264fdc4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/5a7167ecb37e4814944b713932ec3ba1, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/d5a85ab1e3604c59aba39756b7c1a44d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/cfdfa0aba8114ab5913a3a687a40dd63, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/d681d4f74ec74c44a9bb97c59f1d0aa9, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/6e756ae1cdad49f0b2a7a3fbe89cf4ab, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/6cbdfcd187394e26853ec4993d78f3f9, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/ce83d450b692454ba9a0cc54d3c8fe0a] to archive 2024-11-09T02:21:47,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T02:21:47,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/f6475ea7cf934960868b5fa942b61524 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/f6475ea7cf934960868b5fa942b61524 2024-11-09T02:21:47,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/59a3c3e9b11e43d19e54750a4293caa8 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/59a3c3e9b11e43d19e54750a4293caa8 2024-11-09T02:21:47,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/aa09c79478cb4179b4fe6ac78b01099d to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/aa09c79478cb4179b4fe6ac78b01099d 2024-11-09T02:21:47,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/1159651d53b4478d800acf8b8e595c57 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/1159651d53b4478d800acf8b8e595c57 2024-11-09T02:21:47,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/a17e61f611454578a4272576a50c13c4 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/a17e61f611454578a4272576a50c13c4 2024-11-09T02:21:47,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/591bdcac8caf45e3b7b8718cbfbb7470 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/591bdcac8caf45e3b7b8718cbfbb7470 2024-11-09T02:21:47,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/dc3615df843a47e893aad8665efa9e79 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/dc3615df843a47e893aad8665efa9e79 2024-11-09T02:21:47,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/1aae7a0b1a5646dc9e463b8ae2db7978 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/1aae7a0b1a5646dc9e463b8ae2db7978 2024-11-09T02:21:47,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/efa69af4df7743a1a2f4bc5a2ea59d5b to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/efa69af4df7743a1a2f4bc5a2ea59d5b 2024-11-09T02:21:47,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/915922a86d1942b2bdcdfca812683c8b to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/915922a86d1942b2bdcdfca812683c8b 2024-11-09T02:21:47,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/85f2d4a2f325444faae15a5f06f2e3aa to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/85f2d4a2f325444faae15a5f06f2e3aa 2024-11-09T02:21:47,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/ab625d01979d425fa63b0da3886ecf06 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/ab625d01979d425fa63b0da3886ecf06 2024-11-09T02:21:47,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/16873e7b44f64cd5850a010084a41617 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/16873e7b44f64cd5850a010084a41617 2024-11-09T02:21:47,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/b513b3bb91ec4dbc99c32afe637f9546 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/b513b3bb91ec4dbc99c32afe637f9546 2024-11-09T02:21:47,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/06875fe74110448785b4cafa557261c9 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/06875fe74110448785b4cafa557261c9 2024-11-09T02:21:47,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/a20582a74fe941838615457545d0e9cc to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/a20582a74fe941838615457545d0e9cc 2024-11-09T02:21:47,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/389b932d05e1435086771861cbba5ab8 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/389b932d05e1435086771861cbba5ab8 2024-11-09T02:21:47,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/8ef68d74652847d9a30e9679a1ae38e3 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/8ef68d74652847d9a30e9679a1ae38e3 2024-11-09T02:21:47,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/fb2cc24c5c2c4207afe9e07ee19db76b to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/fb2cc24c5c2c4207afe9e07ee19db76b 2024-11-09T02:21:47,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/e28835f448e3485c953ba2ad78a40e91 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/e28835f448e3485c953ba2ad78a40e91 2024-11-09T02:21:47,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/08c51ce536a64d578e81ee34c6e59ee7 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/08c51ce536a64d578e81ee34c6e59ee7 2024-11-09T02:21:47,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/ad024116184d413fa92479f04c165073 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/ad024116184d413fa92479f04c165073 2024-11-09T02:21:47,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/89a52f0446e742bd8645cf9668d8e7b2 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/89a52f0446e742bd8645cf9668d8e7b2 2024-11-09T02:21:47,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/59456abaa30e443c922e135ed06dfc15 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/59456abaa30e443c922e135ed06dfc15 2024-11-09T02:21:47,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/5adc6e3199e74cae95e6e52d8c7496f5 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/5adc6e3199e74cae95e6e52d8c7496f5 2024-11-09T02:21:47,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/743e6ffe702c4ccc921be2366264fdc4 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/743e6ffe702c4ccc921be2366264fdc4 2024-11-09T02:21:47,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/5a7167ecb37e4814944b713932ec3ba1 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/5a7167ecb37e4814944b713932ec3ba1 2024-11-09T02:21:47,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/d5a85ab1e3604c59aba39756b7c1a44d to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/d5a85ab1e3604c59aba39756b7c1a44d 2024-11-09T02:21:47,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/cfdfa0aba8114ab5913a3a687a40dd63 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/cfdfa0aba8114ab5913a3a687a40dd63 2024-11-09T02:21:47,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/d681d4f74ec74c44a9bb97c59f1d0aa9 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/d681d4f74ec74c44a9bb97c59f1d0aa9 2024-11-09T02:21:47,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/6e756ae1cdad49f0b2a7a3fbe89cf4ab to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/6e756ae1cdad49f0b2a7a3fbe89cf4ab 2024-11-09T02:21:47,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/6cbdfcd187394e26853ec4993d78f3f9 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/6cbdfcd187394e26853ec4993d78f3f9 2024-11-09T02:21:47,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/ce83d450b692454ba9a0cc54d3c8fe0a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/ce83d450b692454ba9a0cc54d3c8fe0a 2024-11-09T02:21:47,591 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/recovered.edits/501.seqid, newMaxSeqId=501, maxSeqId=1 2024-11-09T02:21:47,592 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. 2024-11-09T02:21:47,592 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for aad4fc6835c8f9bd4339c4a54a2c56fe: Waiting for close lock at 1731118906116Running coprocessor pre-close hooks at 1731118906116Disabling compacts and flushes for region at 1731118906116Disabling writes for close at 1731118906116Obtaining lock to block concurrent updates at 1731118906117 (+1 ms)Preparing flush snapshotting stores in aad4fc6835c8f9bd4339c4a54a2c56fe at 1731118906117Finished memstore snapshotting TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731118906117Flushing stores of TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe. at 1731118906117Flushing aad4fc6835c8f9bd4339c4a54a2c56fe/A: creating writer at 1731118906118 (+1 ms)Flushing aad4fc6835c8f9bd4339c4a54a2c56fe/A: appending metadata at 1731118906121 (+3 ms)Flushing aad4fc6835c8f9bd4339c4a54a2c56fe/A: closing flushed file at 1731118906121Flushing aad4fc6835c8f9bd4339c4a54a2c56fe/B: creating writer at 1731118906548 (+427 ms)Flushing aad4fc6835c8f9bd4339c4a54a2c56fe/B: appending metadata at 1731118906550 (+2 ms)Flushing aad4fc6835c8f9bd4339c4a54a2c56fe/B: closing flushed file at 1731118906551 (+1 ms)Flushing aad4fc6835c8f9bd4339c4a54a2c56fe/C: creating writer at 1731118906960 (+409 ms)Flushing aad4fc6835c8f9bd4339c4a54a2c56fe/C: appending metadata at 1731118906963 (+3 ms)Flushing aad4fc6835c8f9bd4339c4a54a2c56fe/C: closing flushed file at 1731118906963Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ea82ad: reopening flushed file at 1731118907379 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26ed2c18: reopening flushed file at 1731118907387 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42098fb9: reopening flushed file at 1731118907393 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for aad4fc6835c8f9bd4339c4a54a2c56fe in 1283ms, sequenceid=498, compaction requested=true at 1731118907399 (+6 ms)Writing region close event to WAL at 1731118907587 (+188 ms)Running coprocessor post-close hooks at 1731118907592 (+5 ms)Closed at 1731118907592 2024-11-09T02:21:47,594 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:47,595 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=aad4fc6835c8f9bd4339c4a54a2c56fe, regionState=CLOSED 2024-11-09T02:21:47,597 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 because future has completed 2024-11-09T02:21:47,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-09T02:21:47,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure aad4fc6835c8f9bd4339c4a54a2c56fe, server=53f286d3af3b,46079,1731118875223 in 1.6410 sec 2024-11-09T02:21:47,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-09T02:21:47,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aad4fc6835c8f9bd4339c4a54a2c56fe, UNASSIGN in 1.6500 sec 2024-11-09T02:21:47,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-09T02:21:47,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6580 sec 2024-11-09T02:21:47,611 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118907610"}]},"ts":"1731118907610"} 2024-11-09T02:21:47,613 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T02:21:47,613 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T02:21:47,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6760 sec 2024-11-09T02:21:48,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T02:21:48,101 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:21:48,104 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T02:21:48,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:21:48,110 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:21:48,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-09T02:21:48,112 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:21:48,117 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:48,121 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A, FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B, FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C, FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/recovered.edits] 2024-11-09T02:21:48,124 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/ab76d86317534979b74a53b0054edfce to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/ab76d86317534979b74a53b0054edfce 2024-11-09T02:21:48,125 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/b211d95fcc4f487e9b823edd7fc3a36f to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/b211d95fcc4f487e9b823edd7fc3a36f 2024-11-09T02:21:48,126 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/fb44a7c3f9b34af29cb40e7c985baa94 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/A/fb44a7c3f9b34af29cb40e7c985baa94 2024-11-09T02:21:48,129 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/5ffac1f0926944db815362b8aabbcd4a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/5ffac1f0926944db815362b8aabbcd4a 2024-11-09T02:21:48,130 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/a4524aa67f464ffeaf5e8264c88840df to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/a4524aa67f464ffeaf5e8264c88840df 2024-11-09T02:21:48,131 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/f03f6d2daf5949a7b8b0e17f636c2d20 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/B/f03f6d2daf5949a7b8b0e17f636c2d20 2024-11-09T02:21:48,133 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/0375e862fe5946b9b4b5bd742436aa99 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/0375e862fe5946b9b4b5bd742436aa99 2024-11-09T02:21:48,135 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/2c7170234dd74304933ba00989c9b5e6 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/2c7170234dd74304933ba00989c9b5e6 2024-11-09T02:21:48,136 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/b39853b4a0b94241bc7609a4fdb70847 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/C/b39853b4a0b94241bc7609a4fdb70847 2024-11-09T02:21:48,139 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/recovered.edits/501.seqid to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe/recovered.edits/501.seqid 2024-11-09T02:21:48,139 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/aad4fc6835c8f9bd4339c4a54a2c56fe 2024-11-09T02:21:48,139 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T02:21:48,144 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:21:48,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-09T02:21:48,155 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T02:21:48,185 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T02:21:48,187 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:21:48,187 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T02:21:48,187 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731118908187"}]},"ts":"9223372036854775807"} 2024-11-09T02:21:48,192 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T02:21:48,192 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => aad4fc6835c8f9bd4339c4a54a2c56fe, NAME => 'TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T02:21:48,192 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T02:21:48,192 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731118908192"}]},"ts":"9223372036854775807"} 2024-11-09T02:21:48,195 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T02:21:48,196 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:21:48,198 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 91 msec 2024-11-09T02:21:48,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-09T02:21:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T02:21:48,221 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:21:48,238 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=237 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;53f286d3af3b:46079-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1192768363_22 at /127.0.0.1:33354 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=613 (was 474) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3184 (was 2421) - AvailableMemoryMB LEAK? - 2024-11-09T02:21:48,250 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=613, ProcessCount=11, AvailableMemoryMB=3182 2024-11-09T02:21:48,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T02:21:48,253 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:21:48,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T02:21:48,256 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:21:48,256 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:48,256 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-09T02:21:48,258 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:21:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-09T02:21:48,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741945_1121 (size=959) 2024-11-09T02:21:48,269 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a 2024-11-09T02:21:48,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741946_1122 (size=53) 2024-11-09T02:21:48,278 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:21:48,278 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6ec486933158f26905c39a8004ccca04, disabling compactions & flushes 2024-11-09T02:21:48,278 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:48,278 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:48,278 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. after waiting 0 ms 2024-11-09T02:21:48,278 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:48,278 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:48,278 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6ec486933158f26905c39a8004ccca04: Waiting for close lock at 1731118908278Disabling compacts and flushes for region at 1731118908278Disabling writes for close at 1731118908278Writing region close event to WAL at 1731118908278Closed at 1731118908278 2024-11-09T02:21:48,279 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:21:48,280 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731118908279"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118908279"}]},"ts":"1731118908279"} 2024-11-09T02:21:48,282 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T02:21:48,287 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:21:48,287 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118908287"}]},"ts":"1731118908287"} 2024-11-09T02:21:48,289 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T02:21:48,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ec486933158f26905c39a8004ccca04, ASSIGN}] 2024-11-09T02:21:48,291 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ec486933158f26905c39a8004ccca04, ASSIGN 2024-11-09T02:21:48,292 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ec486933158f26905c39a8004ccca04, ASSIGN; state=OFFLINE, location=53f286d3af3b,46079,1731118875223; forceNewPlan=false, retain=false 2024-11-09T02:21:48,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-09T02:21:48,443 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=6ec486933158f26905c39a8004ccca04, regionState=OPENING, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:48,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ec486933158f26905c39a8004ccca04, ASSIGN because future has completed 2024-11-09T02:21:48,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223}] 2024-11-09T02:21:48,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-09T02:21:48,602 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:48,603 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:21:48,603 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:48,603 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:21:48,603 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:48,603 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:48,605 INFO [StoreOpener-6ec486933158f26905c39a8004ccca04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:48,607 INFO [StoreOpener-6ec486933158f26905c39a8004ccca04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ec486933158f26905c39a8004ccca04 columnFamilyName A 2024-11-09T02:21:48,607 DEBUG [StoreOpener-6ec486933158f26905c39a8004ccca04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:48,608 INFO [StoreOpener-6ec486933158f26905c39a8004ccca04-1 {}] regionserver.HStore(327): Store=6ec486933158f26905c39a8004ccca04/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:21:48,608 INFO [StoreOpener-6ec486933158f26905c39a8004ccca04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:48,610 INFO [StoreOpener-6ec486933158f26905c39a8004ccca04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ec486933158f26905c39a8004ccca04 columnFamilyName B 2024-11-09T02:21:48,611 DEBUG [StoreOpener-6ec486933158f26905c39a8004ccca04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:48,611 INFO [StoreOpener-6ec486933158f26905c39a8004ccca04-1 {}] regionserver.HStore(327): Store=6ec486933158f26905c39a8004ccca04/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:21:48,611 INFO [StoreOpener-6ec486933158f26905c39a8004ccca04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:48,613 INFO [StoreOpener-6ec486933158f26905c39a8004ccca04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ec486933158f26905c39a8004ccca04 columnFamilyName C 2024-11-09T02:21:48,613 DEBUG [StoreOpener-6ec486933158f26905c39a8004ccca04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:48,613 INFO [StoreOpener-6ec486933158f26905c39a8004ccca04-1 {}] regionserver.HStore(327): Store=6ec486933158f26905c39a8004ccca04/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:21:48,614 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:48,614 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:48,615 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:48,616 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:48,616 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:48,617 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T02:21:48,618 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:48,620 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:21:48,621 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 6ec486933158f26905c39a8004ccca04; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63128403, jitterRate=-0.059313490986824036}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T02:21:48,621 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:48,622 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 6ec486933158f26905c39a8004ccca04: Running coprocessor pre-open hook at 1731118908604Writing region info on filesystem at 1731118908604Initializing all the Stores at 1731118908605 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118908605Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118908605Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118908605Cleaning up temporary data from old regions at 1731118908616 (+11 ms)Running coprocessor post-open hooks at 1731118908621 (+5 ms)Region opened successfully at 1731118908622 (+1 ms) 2024-11-09T02:21:48,623 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., pid=32, masterSystemTime=1731118908598 2024-11-09T02:21:48,626 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:48,626 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:48,626 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=6ec486933158f26905c39a8004ccca04, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:48,628 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 because future has completed 2024-11-09T02:21:48,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-09T02:21:48,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 in 183 msec 2024-11-09T02:21:48,633 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-09T02:21:48,633 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ec486933158f26905c39a8004ccca04, ASSIGN in 341 msec 2024-11-09T02:21:48,634 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:21:48,635 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118908635"}]},"ts":"1731118908635"} 2024-11-09T02:21:48,637 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T02:21:48,638 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:21:48,640 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 385 msec 2024-11-09T02:21:48,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-09T02:21:48,891 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:21:48,892 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:48,892 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:21:48,892 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:21:48,894 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:21:48,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:21:48,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:21:48,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@595cff53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:48,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:21:48,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:21:48,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:48,896 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:21:48,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a1c705, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:48,898 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:21:48,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:21:48,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:48,901 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:48,902 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T02:21:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 53f286d3af3b,45041,1731118874457 2024-11-09T02:21:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@234e638 2024-11-09T02:21:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T02:21:48,905 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T02:21:48,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T02:21:48,910 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:21:48,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-09T02:21:48,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741947_1123 (size=995) 2024-11-09T02:21:49,334 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-09T02:21:49,334 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-09T02:21:49,341 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T02:21:49,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ec486933158f26905c39a8004ccca04, REOPEN/MOVE}] 2024-11-09T02:21:49,349 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ec486933158f26905c39a8004ccca04, REOPEN/MOVE 2024-11-09T02:21:49,349 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=6ec486933158f26905c39a8004ccca04, regionState=CLOSING, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:49,351 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ec486933158f26905c39a8004ccca04, REOPEN/MOVE because future has completed 2024-11-09T02:21:49,351 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T02:21:49,351 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223}] 2024-11-09T02:21:49,505 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:49,506 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T02:21:49,506 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 6ec486933158f26905c39a8004ccca04, disabling compactions & flushes 2024-11-09T02:21:49,506 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:49,506 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:49,506 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. after waiting 0 ms 2024-11-09T02:21:49,506 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:49,514 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:21:49,515 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:49,515 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 6ec486933158f26905c39a8004ccca04: Waiting for close lock at 1731118909506Running coprocessor pre-close hooks at 1731118909506Disabling compacts and flushes for region at 1731118909506Disabling writes for close at 1731118909506Writing region close event to WAL at 1731118909507 (+1 ms)Running coprocessor post-close hooks at 1731118909515 (+8 ms)Closed at 1731118909515 2024-11-09T02:21:49,515 WARN [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 6ec486933158f26905c39a8004ccca04 to self. 2024-11-09T02:21:49,518 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:49,519 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=6ec486933158f26905c39a8004ccca04, regionState=CLOSED 2024-11-09T02:21:49,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 because future has completed 2024-11-09T02:21:49,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-09T02:21:49,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 in 171 msec 2024-11-09T02:21:49,525 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ec486933158f26905c39a8004ccca04, REOPEN/MOVE; state=CLOSED, location=53f286d3af3b,46079,1731118875223; forceNewPlan=false, retain=true 2024-11-09T02:21:49,676 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=6ec486933158f26905c39a8004ccca04, regionState=OPENING, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:49,678 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ec486933158f26905c39a8004ccca04, REOPEN/MOVE because future has completed 2024-11-09T02:21:49,678 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223}] 2024-11-09T02:21:49,836 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:49,836 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:21:49,837 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:49,837 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:21:49,837 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:49,838 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:49,844 INFO [StoreOpener-6ec486933158f26905c39a8004ccca04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:49,849 INFO [StoreOpener-6ec486933158f26905c39a8004ccca04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ec486933158f26905c39a8004ccca04 columnFamilyName A 2024-11-09T02:21:49,851 DEBUG [StoreOpener-6ec486933158f26905c39a8004ccca04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:49,851 INFO [StoreOpener-6ec486933158f26905c39a8004ccca04-1 {}] regionserver.HStore(327): Store=6ec486933158f26905c39a8004ccca04/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:21:49,852 INFO [StoreOpener-6ec486933158f26905c39a8004ccca04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:49,852 INFO [StoreOpener-6ec486933158f26905c39a8004ccca04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ec486933158f26905c39a8004ccca04 columnFamilyName B 2024-11-09T02:21:49,852 DEBUG [StoreOpener-6ec486933158f26905c39a8004ccca04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:49,853 INFO [StoreOpener-6ec486933158f26905c39a8004ccca04-1 {}] regionserver.HStore(327): Store=6ec486933158f26905c39a8004ccca04/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:21:49,853 INFO [StoreOpener-6ec486933158f26905c39a8004ccca04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:49,853 INFO [StoreOpener-6ec486933158f26905c39a8004ccca04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ec486933158f26905c39a8004ccca04 columnFamilyName C 2024-11-09T02:21:49,853 DEBUG [StoreOpener-6ec486933158f26905c39a8004ccca04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:49,854 INFO [StoreOpener-6ec486933158f26905c39a8004ccca04-1 {}] regionserver.HStore(327): Store=6ec486933158f26905c39a8004ccca04/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:21:49,854 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:49,855 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:49,856 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:49,857 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:49,857 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:49,857 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T02:21:49,858 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:49,859 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 6ec486933158f26905c39a8004ccca04; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69789612, jitterRate=0.03994625806808472}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T02:21:49,859 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:49,861 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 6ec486933158f26905c39a8004ccca04: Running coprocessor pre-open hook at 1731118909838Writing region info on filesystem at 1731118909838Initializing all the Stores at 1731118909839 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118909839Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118909840 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118909840Cleaning up temporary data from old regions at 1731118909857 (+17 ms)Running coprocessor post-open hooks at 1731118909859 (+2 ms)Region opened successfully at 1731118909861 (+2 ms) 2024-11-09T02:21:49,862 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., pid=37, masterSystemTime=1731118909831 2024-11-09T02:21:49,864 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:49,864 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:49,865 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=6ec486933158f26905c39a8004ccca04, regionState=OPEN, openSeqNum=5, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:49,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 because future has completed 2024-11-09T02:21:49,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-09T02:21:49,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 in 189 msec 2024-11-09T02:21:49,871 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-09T02:21:49,871 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ec486933158f26905c39a8004ccca04, REOPEN/MOVE in 521 msec 2024-11-09T02:21:49,874 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-09T02:21:49,874 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 532 msec 2024-11-09T02:21:49,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 962 msec 2024-11-09T02:21:49,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-09T02:21:49,878 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:21:49,879 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,879 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:21:49,879 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:21:49,880 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:21:49,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:21:49,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:21:49,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201cdbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:21:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:21:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:49,882 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:21:49,882 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f07f36a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,883 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:21:49,883 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:21:49,884 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:21:49,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:21:49,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:21:49,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3035364d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:21:49,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:21:49,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:49,885 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:21:49,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b238d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,886 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,886 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:21:49,886 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:21:49,887 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:21:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:21:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:21:49,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c51e436, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:21:49,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:21:49,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:49,889 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:21:49,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12b0cd7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,890 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,890 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:21:49,891 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:21:49,891 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:21:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:21:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:21:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42c8a135, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:21:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:21:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:49,893 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:21:49,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12627dfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,894 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:21:49,894 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:21:49,895 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:21:49,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:21:49,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:21:49,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f1821ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:21:49,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:21:49,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:49,897 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:21:49,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a87d6bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,898 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:21:49,899 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:21:49,899 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:21:49,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:21:49,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:21:49,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@505721e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:21:49,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:21:49,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:49,901 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:21:49,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61e330ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,902 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:21:49,902 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:21:49,903 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:21:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:21:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:21:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5395290f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:21:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:21:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:49,904 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:21:49,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21253c62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,905 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:21:49,905 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:21:49,906 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:21:49,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:21:49,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:21:49,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67fa0b36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:21:49,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:21:49,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:49,907 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:21:49,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cdff758, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,909 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,909 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:21:49,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:21:49,910 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:21:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:21:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:21:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3654db63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:21:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:21:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:49,912 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:21:49,912 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bb02826, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:49,913 DEBUG [Thread-632 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:21:49,913 DEBUG [Thread-634 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:21:49,913 DEBUG [Thread-638 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:21:49,913 DEBUG [Thread-625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:21:49,913 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:21:49,914 DEBUG [Thread-623 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:21:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:21:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:21:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-09T02:21:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:21:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:21:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:21:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-09T02:21:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:49,915 DEBUG [Thread-629 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:21:49,916 DEBUG [Thread-636 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:21:49,916 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:21:49,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:49,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:49,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:49,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:49,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:21:49,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:49,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:49,917 DEBUG [Thread-627 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:21:49,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:49,918 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:21:49,918 DEBUG [Thread-629 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:21:49,918 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:21:49,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:49,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:21:49,918 DEBUG [Thread-629 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:49,919 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:49,919 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:21:49,921 DEBUG [Thread-621 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:21:49,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:21:49,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:21:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:21:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:21:49,925 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:21:49,925 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:49,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:21:49,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:21:49,927 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:49,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:21:49,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:21:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:49,941 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:21:49,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ec486933158f26905c39a8004ccca04 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T02:21:49,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:49,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118969979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:49,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:49,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:49,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:49,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118969981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:49,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:49,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:49,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:49,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118969988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:49,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118969990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:49,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:49,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:49,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118969994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:49,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:49,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:49,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:49,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:49,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:49,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:49,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:50,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110988e574bd200f401987680ca346d779ed_6ec486933158f26905c39a8004ccca04 is 50, key is test_row_0/A:col10/1731118909939/Put/seqid=0 2024-11-09T02:21:50,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-09T02:21:50,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741948_1124 (size=12154) 2024-11-09T02:21:50,040 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:50,048 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110988e574bd200f401987680ca346d779ed_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110988e574bd200f401987680ca346d779ed_6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:50,050 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/79ccbf69fd5c42f9acd2d9546cb5168c, store: [table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:21:50,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/79ccbf69fd5c42f9acd2d9546cb5168c is 175, key is test_row_0/A:col10/1731118909939/Put/seqid=0 2024-11-09T02:21:50,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741949_1125 (size=30955) 2024-11-09T02:21:50,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T02:21:50,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:50,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:21:50,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:50,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:50,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:50,075 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/79ccbf69fd5c42f9acd2d9546cb5168c 2024-11-09T02:21:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:50,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118970101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:50,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:50,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:50,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118970101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118970101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:50,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:50,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:50,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:50,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/2f3d04ad3cd3438ebb75149004e5dad3 is 50, key is test_row_0/B:col10/1731118909939/Put/seqid=0 2024-11-09T02:21:50,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118970101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:50,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118970111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:50,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741950_1126 (size=12001) 2024-11-09T02:21:50,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/2f3d04ad3cd3438ebb75149004e5dad3 2024-11-09T02:21:50,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/0fdb1985042e412d955872cc8fc8017d is 50, key is test_row_0/C:col10/1731118909939/Put/seqid=0 2024-11-09T02:21:50,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741951_1127 (size=12001) 2024-11-09T02:21:50,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T02:21:50,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-09T02:21:50,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:50,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:21:50,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:50,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:50,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:50,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:50,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:50,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118970310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:50,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:50,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118970311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:50,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:50,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118970311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:50,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:50,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:50,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118970320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:50,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:50,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118970320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:50,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T02:21:50,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:50,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:21:50,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:50,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:50,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:50,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:50,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T02:21:50,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:50,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:21:50,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:50,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:50,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:50,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:50,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-09T02:21:50,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/0fdb1985042e412d955872cc8fc8017d 2024-11-09T02:21:50,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:50,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118970621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:50,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:50,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118970621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:50,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:50,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:50,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118970621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:50,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:50,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:50,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/79ccbf69fd5c42f9acd2d9546cb5168c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/79ccbf69fd5c42f9acd2d9546cb5168c 2024-11-09T02:21:50,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:50,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118970630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:50,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:50,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:50,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118970630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:50,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:50,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:50,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/79ccbf69fd5c42f9acd2d9546cb5168c, entries=150, sequenceid=16, filesize=30.2 K 2024-11-09T02:21:50,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/2f3d04ad3cd3438ebb75149004e5dad3 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/2f3d04ad3cd3438ebb75149004e5dad3 2024-11-09T02:21:50,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/2f3d04ad3cd3438ebb75149004e5dad3, entries=150, sequenceid=16, filesize=11.7 K 2024-11-09T02:21:50,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/0fdb1985042e412d955872cc8fc8017d as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/0fdb1985042e412d955872cc8fc8017d 2024-11-09T02:21:50,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/0fdb1985042e412d955872cc8fc8017d, entries=150, sequenceid=16, filesize=11.7 K 2024-11-09T02:21:50,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6ec486933158f26905c39a8004ccca04 in 721ms, sequenceid=16, compaction requested=false 2024-11-09T02:21:50,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:21:50,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T02:21:50,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:50,692 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 6ec486933158f26905c39a8004ccca04 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-09T02:21:50,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ccd2c8fbd32c4ac4a27459a0a4356895_6ec486933158f26905c39a8004ccca04 is 50, key is test_row_0/A:col10/1731118909991/Put/seqid=0 2024-11-09T02:21:50,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741952_1128 (size=12154) 2024-11-09T02:21:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-09T02:21:51,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:51,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:21:51,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:51,139 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ccd2c8fbd32c4ac4a27459a0a4356895_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ccd2c8fbd32c4ac4a27459a0a4356895_6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:51,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/8c492854e0b74361b0ea1f9daaaa026e, store: [table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:21:51,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/8c492854e0b74361b0ea1f9daaaa026e is 175, key is test_row_0/A:col10/1731118909991/Put/seqid=0 2024-11-09T02:21:51,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741953_1129 (size=30955) 2024-11-09T02:21:51,150 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/8c492854e0b74361b0ea1f9daaaa026e 2024-11-09T02:21:51,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/aad047c6aef54698aa5e93e4d363f3eb is 50, key is test_row_0/B:col10/1731118909991/Put/seqid=0 2024-11-09T02:21:51,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741954_1130 (size=12001) 2024-11-09T02:21:51,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118971176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:51,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118971176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:51,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118971176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:51,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118971178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:51,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:51,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118971178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:51,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:51,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:51,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118971289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:51,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:51,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118971290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:51,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:51,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118971291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:51,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:51,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118971291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:51,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:51,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:51,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118971292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:51,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:51,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118971500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:51,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:51,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118971501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:51,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:51,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:51,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118971501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:51,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:51,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118971501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:51,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118971504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:51,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:51,571 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/aad047c6aef54698aa5e93e4d363f3eb 2024-11-09T02:21:51,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/5cda97cc16074df1b4bb7a8d64d60135 is 50, key is test_row_0/C:col10/1731118909991/Put/seqid=0 2024-11-09T02:21:51,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741955_1131 (size=12001) 2024-11-09T02:21:51,584 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/5cda97cc16074df1b4bb7a8d64d60135 2024-11-09T02:21:51,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/8c492854e0b74361b0ea1f9daaaa026e as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/8c492854e0b74361b0ea1f9daaaa026e 2024-11-09T02:21:51,600 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/8c492854e0b74361b0ea1f9daaaa026e, entries=150, sequenceid=40, filesize=30.2 K 2024-11-09T02:21:51,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/aad047c6aef54698aa5e93e4d363f3eb as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/aad047c6aef54698aa5e93e4d363f3eb 2024-11-09T02:21:51,608 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/aad047c6aef54698aa5e93e4d363f3eb, entries=150, sequenceid=40, filesize=11.7 K 2024-11-09T02:21:51,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/5cda97cc16074df1b4bb7a8d64d60135 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/5cda97cc16074df1b4bb7a8d64d60135 2024-11-09T02:21:51,628 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/5cda97cc16074df1b4bb7a8d64d60135, entries=150, sequenceid=40, filesize=11.7 K 2024-11-09T02:21:51,630 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 6ec486933158f26905c39a8004ccca04 in 938ms, sequenceid=40, compaction requested=false 2024-11-09T02:21:51,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:21:51,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:51,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-09T02:21:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-09T02:21:51,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-09T02:21:51,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7130 sec 2024-11-09T02:21:51,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.7240 sec 2024-11-09T02:21:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:51,813 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:21:51,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ec486933158f26905c39a8004ccca04 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-09T02:21:51,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110996ffc92daac24c2892678080c2f14fac_6ec486933158f26905c39a8004ccca04 is 50, key is test_row_0/A:col10/1731118911812/Put/seqid=0 2024-11-09T02:21:51,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118971852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:51,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:51,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118971854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:51,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:51,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118971859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:51,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118971859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:51,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118971860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:51,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:51,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:51,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:51,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741956_1132 (size=12154) 2024-11-09T02:21:51,883 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:51,896 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110996ffc92daac24c2892678080c2f14fac_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110996ffc92daac24c2892678080c2f14fac_6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:51,898 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/d02e875e8e96433eb0baa40027ee8c6b, store: [table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:21:51,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/d02e875e8e96433eb0baa40027ee8c6b is 175, key is test_row_0/A:col10/1731118911812/Put/seqid=0 2024-11-09T02:21:51,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741957_1133 (size=30955) 2024-11-09T02:21:51,921 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/d02e875e8e96433eb0baa40027ee8c6b 2024-11-09T02:21:51,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/bc393ce77e554fcda79fa8873c8bcb19 is 50, key is test_row_0/B:col10/1731118911812/Put/seqid=0 2024-11-09T02:21:51,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741958_1134 (size=12001) 2024-11-09T02:21:51,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/bc393ce77e554fcda79fa8873c8bcb19 2024-11-09T02:21:51,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118971970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:51,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118971970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:51,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:51,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118971971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:51,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:51,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:51,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:51,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118971970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:51,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/268264bdc7634397a187f60f5f62cb05 is 50, key is test_row_0/C:col10/1731118911812/Put/seqid=0 2024-11-09T02:21:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118971972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:51,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:51,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:51,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741959_1135 (size=12001) 2024-11-09T02:21:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-09T02:21:52,070 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:21:52,071 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:21:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-09T02:21:52,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T02:21:52,074 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:21:52,075 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:21:52,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:21:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T02:21:52,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118972180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:52,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:52,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118972180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:52,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:52,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118972181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:52,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:52,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:52,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118972181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118972181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:52,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:52,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:52,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:52,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T02:21:52,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:52,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:21:52,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:52,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:52,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:52,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T02:21:52,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:52,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:21:52,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:52,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:52,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T02:21:52,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/268264bdc7634397a187f60f5f62cb05 2024-11-09T02:21:52,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/d02e875e8e96433eb0baa40027ee8c6b as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/d02e875e8e96433eb0baa40027ee8c6b 2024-11-09T02:21:52,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/d02e875e8e96433eb0baa40027ee8c6b, entries=150, sequenceid=55, filesize=30.2 K 2024-11-09T02:21:52,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/bc393ce77e554fcda79fa8873c8bcb19 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/bc393ce77e554fcda79fa8873c8bcb19 2024-11-09T02:21:52,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/bc393ce77e554fcda79fa8873c8bcb19, entries=150, sequenceid=55, filesize=11.7 K 2024-11-09T02:21:52,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/268264bdc7634397a187f60f5f62cb05 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/268264bdc7634397a187f60f5f62cb05 2024-11-09T02:21:52,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/268264bdc7634397a187f60f5f62cb05, entries=150, sequenceid=55, filesize=11.7 K 2024-11-09T02:21:52,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 6ec486933158f26905c39a8004ccca04 in 627ms, sequenceid=55, compaction requested=true 2024-11-09T02:21:52,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:21:52,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ec486933158f26905c39a8004ccca04:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:21:52,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:52,440 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:52,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ec486933158f26905c39a8004ccca04:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:21:52,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:52,440 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:52,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ec486933158f26905c39a8004ccca04:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:21:52,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:21:52,442 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:52,442 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 6ec486933158f26905c39a8004ccca04/B is initiating minor compaction (all files) 2024-11-09T02:21:52,442 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ec486933158f26905c39a8004ccca04/B in TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:52,442 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/2f3d04ad3cd3438ebb75149004e5dad3, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/aad047c6aef54698aa5e93e4d363f3eb, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/bc393ce77e554fcda79fa8873c8bcb19] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp, totalSize=35.2 K 2024-11-09T02:21:52,442 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f3d04ad3cd3438ebb75149004e5dad3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731118909933 2024-11-09T02:21:52,443 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting aad047c6aef54698aa5e93e4d363f3eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731118909978 2024-11-09T02:21:52,443 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:52,443 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): 6ec486933158f26905c39a8004ccca04/A is initiating minor compaction (all files) 2024-11-09T02:21:52,443 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ec486933158f26905c39a8004ccca04/A in TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:52,443 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting bc393ce77e554fcda79fa8873c8bcb19, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731118911175 2024-11-09T02:21:52,443 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/79ccbf69fd5c42f9acd2d9546cb5168c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/8c492854e0b74361b0ea1f9daaaa026e, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/d02e875e8e96433eb0baa40027ee8c6b] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp, totalSize=90.7 K 2024-11-09T02:21:52,443 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:52,443 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. files: [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/79ccbf69fd5c42f9acd2d9546cb5168c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/8c492854e0b74361b0ea1f9daaaa026e, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/d02e875e8e96433eb0baa40027ee8c6b] 2024-11-09T02:21:52,444 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79ccbf69fd5c42f9acd2d9546cb5168c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731118909933 2024-11-09T02:21:52,444 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c492854e0b74361b0ea1f9daaaa026e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731118909978 2024-11-09T02:21:52,445 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting d02e875e8e96433eb0baa40027ee8c6b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731118911175 2024-11-09T02:21:52,452 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ec486933158f26905c39a8004ccca04#B#compaction#117 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:52,453 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/19ca8b3d2ca6438b93e9eac575dba010 is 50, key is test_row_0/B:col10/1731118911812/Put/seqid=0 2024-11-09T02:21:52,461 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:21:52,465 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109d28707f6fc9744aa9ec4935a867e29a4_6ec486933158f26905c39a8004ccca04 store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:21:52,472 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109d28707f6fc9744aa9ec4935a867e29a4_6ec486933158f26905c39a8004ccca04, store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:21:52,473 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d28707f6fc9744aa9ec4935a867e29a4_6ec486933158f26905c39a8004ccca04 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:21:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:52,495 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:21:52,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ec486933158f26905c39a8004ccca04 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-09T02:21:52,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741960_1136 (size=12104) 2024-11-09T02:21:52,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118972506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:52,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118972507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:52,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118972508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118972508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:52,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:52,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:52,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118972514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:52,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:52,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741961_1137 (size=4469) 2024-11-09T02:21:52,522 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ec486933158f26905c39a8004ccca04#A#compaction#118 average throughput is 0.40 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:52,523 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/3417402b38db48fea85429e450420471 is 175, key is test_row_0/A:col10/1731118911812/Put/seqid=0 2024-11-09T02:21:52,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T02:21:52,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:52,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:21:52,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:52,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:52,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:52,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741962_1138 (size=31058) 2024-11-09T02:21:52,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110938b6db5120be448c9941a2571df61341_6ec486933158f26905c39a8004ccca04 is 50, key is test_row_0/A:col10/1731118912494/Put/seqid=0 2024-11-09T02:21:52,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741963_1139 (size=17034) 2024-11-09T02:21:52,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118972620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118972620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:52,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118972620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118972621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:52,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:52,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118972631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:52,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T02:21:52,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:52,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:21:52,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:52,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:52,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T02:21:52,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:52,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118972831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118972831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:52,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118972832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:52,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:52,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:52,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:52,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118972833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:52,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:52,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T02:21:52,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:52,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:21:52,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:52,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:52,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:52,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:52,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118972839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:52,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:52,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:52,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:52,913 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/19ca8b3d2ca6438b93e9eac575dba010 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/19ca8b3d2ca6438b93e9eac575dba010 2024-11-09T02:21:52,921 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ec486933158f26905c39a8004ccca04/B of 6ec486933158f26905c39a8004ccca04 into 19ca8b3d2ca6438b93e9eac575dba010(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:52,921 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:21:52,921 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., storeName=6ec486933158f26905c39a8004ccca04/B, priority=13, startTime=1731118912440; duration=0sec 2024-11-09T02:21:52,921 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:21:52,921 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ec486933158f26905c39a8004ccca04:B 2024-11-09T02:21:52,921 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:52,924 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:52,924 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 6ec486933158f26905c39a8004ccca04/C is initiating minor compaction (all files) 2024-11-09T02:21:52,924 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ec486933158f26905c39a8004ccca04/C in TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:52,924 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/0fdb1985042e412d955872cc8fc8017d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/5cda97cc16074df1b4bb7a8d64d60135, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/268264bdc7634397a187f60f5f62cb05] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp, totalSize=35.2 K 2024-11-09T02:21:52,925 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fdb1985042e412d955872cc8fc8017d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731118909933 2024-11-09T02:21:52,925 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cda97cc16074df1b4bb7a8d64d60135, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731118909978 2024-11-09T02:21:52,925 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 268264bdc7634397a187f60f5f62cb05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731118911175 2024-11-09T02:21:52,936 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ec486933158f26905c39a8004ccca04#C#compaction#120 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:52,936 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/7da40e7ddea54667b36e4c84c64dd5a5 is 50, key is test_row_0/C:col10/1731118911812/Put/seqid=0 2024-11-09T02:21:52,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741964_1140 (size=12104) 2024-11-09T02:21:52,952 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/3417402b38db48fea85429e450420471 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/3417402b38db48fea85429e450420471 2024-11-09T02:21:52,960 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:52,967 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110938b6db5120be448c9941a2571df61341_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110938b6db5120be448c9941a2571df61341_6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:52,969 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/d3d9c81e80e94d3b94b5754abdbe6b72, store: [table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:21:52,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/d3d9c81e80e94d3b94b5754abdbe6b72 is 175, key is test_row_0/A:col10/1731118912494/Put/seqid=0 2024-11-09T02:21:52,972 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ec486933158f26905c39a8004ccca04/A of 6ec486933158f26905c39a8004ccca04 into 3417402b38db48fea85429e450420471(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:52,972 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:21:52,972 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., storeName=6ec486933158f26905c39a8004ccca04/A, priority=13, startTime=1731118912440; duration=0sec 2024-11-09T02:21:52,972 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:52,972 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ec486933158f26905c39a8004ccca04:A 2024-11-09T02:21:52,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741965_1141 (size=48139) 2024-11-09T02:21:52,983 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/d3d9c81e80e94d3b94b5754abdbe6b72 2024-11-09T02:21:52,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/341615a7240c47d39e160b5a600c4986 is 50, key is test_row_0/B:col10/1731118912494/Put/seqid=0 2024-11-09T02:21:52,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T02:21:52,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:52,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:21:52,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:52,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:52,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:52,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:53,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741966_1142 (size=12001) 2024-11-09T02:21:53,124 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:21:53,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:53,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118973140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:53,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:53,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:53,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:53,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118973140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:53,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:53,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:53,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118973140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:53,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:53,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118973140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:53,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:53,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T02:21:53,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:53,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:21:53,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:53,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:53,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:53,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:53,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:53,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118973150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:53,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:53,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T02:21:53,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T02:21:53,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:53,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:21:53,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:53,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:53,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:53,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:53,363 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/7da40e7ddea54667b36e4c84c64dd5a5 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/7da40e7ddea54667b36e4c84c64dd5a5 2024-11-09T02:21:53,373 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ec486933158f26905c39a8004ccca04/C of 6ec486933158f26905c39a8004ccca04 into 7da40e7ddea54667b36e4c84c64dd5a5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:53,373 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:21:53,373 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., storeName=6ec486933158f26905c39a8004ccca04/C, priority=13, startTime=1731118912440; duration=0sec 2024-11-09T02:21:53,373 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:53,373 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ec486933158f26905c39a8004ccca04:C 2024-11-09T02:21:53,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/341615a7240c47d39e160b5a600c4986 2024-11-09T02:21:53,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/6c96e7e9e5cd4e1eb115d2f2231aec36 is 50, key is test_row_0/C:col10/1731118912494/Put/seqid=0 2024-11-09T02:21:53,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741967_1143 (size=12001) 2024-11-09T02:21:53,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/6c96e7e9e5cd4e1eb115d2f2231aec36 2024-11-09T02:21:53,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/d3d9c81e80e94d3b94b5754abdbe6b72 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/d3d9c81e80e94d3b94b5754abdbe6b72 2024-11-09T02:21:53,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T02:21:53,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:53,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:21:53,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:53,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:53,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:53,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/d3d9c81e80e94d3b94b5754abdbe6b72, entries=250, sequenceid=79, filesize=47.0 K 2024-11-09T02:21:53,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:53,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/341615a7240c47d39e160b5a600c4986 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/341615a7240c47d39e160b5a600c4986 2024-11-09T02:21:53,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/341615a7240c47d39e160b5a600c4986, entries=150, sequenceid=79, filesize=11.7 K 2024-11-09T02:21:53,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/6c96e7e9e5cd4e1eb115d2f2231aec36 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/6c96e7e9e5cd4e1eb115d2f2231aec36 2024-11-09T02:21:53,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/6c96e7e9e5cd4e1eb115d2f2231aec36, entries=150, sequenceid=79, filesize=11.7 K 2024-11-09T02:21:53,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 6ec486933158f26905c39a8004ccca04 in 985ms, sequenceid=79, compaction requested=false 2024-11-09T02:21:53,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:21:53,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T02:21:53,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:53,632 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 6ec486933158f26905c39a8004ccca04 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T02:21:53,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110949139ba58f394088b700254b9e6b70f7_6ec486933158f26905c39a8004ccca04 is 50, key is test_row_0/A:col10/1731118912506/Put/seqid=0 2024-11-09T02:21:53,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:53,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:21:53,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741968_1144 (size=12154) 2024-11-09T02:21:53,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:53,684 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110949139ba58f394088b700254b9e6b70f7_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110949139ba58f394088b700254b9e6b70f7_6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:53,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/de4c7e99259146519557c6a21cfc99b3, store: [table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:21:53,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/de4c7e99259146519557c6a21cfc99b3 is 175, key is test_row_0/A:col10/1731118912506/Put/seqid=0 2024-11-09T02:21:53,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:53,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118973680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:53,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:53,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:53,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118973687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:53,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:53,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118973689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:53,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118973687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:53,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:53,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118973689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:53,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741969_1145 (size=30955) 2024-11-09T02:21:53,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118973799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:53,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118973799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:53,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118973800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118973800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:53,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118973800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:54,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118974010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:54,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:54,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:54,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118974010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118974010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118974010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:54,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118974010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:54,115 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/de4c7e99259146519557c6a21cfc99b3 2024-11-09T02:21:54,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/a4908276de084821bc2dd66130139244 is 50, key is test_row_0/B:col10/1731118912506/Put/seqid=0 2024-11-09T02:21:54,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741970_1146 (size=12001) 2024-11-09T02:21:54,133 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/a4908276de084821bc2dd66130139244 2024-11-09T02:21:54,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/391676a9990e4c7581d87608c16d1848 is 50, key is test_row_0/C:col10/1731118912506/Put/seqid=0 2024-11-09T02:21:54,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741971_1147 (size=12001) 2024-11-09T02:21:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T02:21:54,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118974320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:54,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:54,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118974320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:54,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:54,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118974321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118974321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118974321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:54,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:54,571 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/391676a9990e4c7581d87608c16d1848 2024-11-09T02:21:54,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/de4c7e99259146519557c6a21cfc99b3 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/de4c7e99259146519557c6a21cfc99b3 2024-11-09T02:21:54,582 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/de4c7e99259146519557c6a21cfc99b3, entries=150, sequenceid=94, filesize=30.2 K 2024-11-09T02:21:54,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/a4908276de084821bc2dd66130139244 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a4908276de084821bc2dd66130139244 2024-11-09T02:21:54,588 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a4908276de084821bc2dd66130139244, entries=150, sequenceid=94, filesize=11.7 K 2024-11-09T02:21:54,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/391676a9990e4c7581d87608c16d1848 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/391676a9990e4c7581d87608c16d1848 2024-11-09T02:21:54,595 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/391676a9990e4c7581d87608c16d1848, entries=150, sequenceid=94, filesize=11.7 K 2024-11-09T02:21:54,596 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 6ec486933158f26905c39a8004ccca04 in 964ms, sequenceid=94, compaction requested=true 2024-11-09T02:21:54,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:21:54,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:54,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-09T02:21:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-09T02:21:54,602 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-09T02:21:54,602 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5220 sec 2024-11-09T02:21:54,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.5320 sec 2024-11-09T02:21:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:54,832 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:21:54,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ec486933158f26905c39a8004ccca04 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-09T02:21:54,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411092f048f1a4d244d81bac7e5176a68ce87_6ec486933158f26905c39a8004ccca04 is 50, key is test_row_0/A:col10/1731118913686/Put/seqid=0 2024-11-09T02:21:54,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118974840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:54,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:54,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741972_1148 (size=14594) 2024-11-09T02:21:54,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118974844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:54,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118974846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:54,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118974844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118974846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:54,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118974960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118974960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:54,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:54,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118974960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:54,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:54,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118974960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:55,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:55,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118975170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:55,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:55,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:55,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:55,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:55,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118975170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:55,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:55,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:55,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:55,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:55,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118975170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:55,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:55,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:55,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:55,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118975170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:55,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:55,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:55,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:55,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:55,248 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:55,252 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411092f048f1a4d244d81bac7e5176a68ce87_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092f048f1a4d244d81bac7e5176a68ce87_6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:55,253 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/3b146ba2d2fa47f1ac0d9f2f8fdc0ed5, store: [table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:21:55,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/3b146ba2d2fa47f1ac0d9f2f8fdc0ed5 is 175, key is test_row_0/A:col10/1731118913686/Put/seqid=0 2024-11-09T02:21:55,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741973_1149 (size=39549) 2024-11-09T02:21:55,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:55,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118975480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118975480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:55,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:55,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118975480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118975480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:55,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:55,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:55,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:55,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:55,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:55,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:55,659 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/3b146ba2d2fa47f1ac0d9f2f8fdc0ed5 2024-11-09T02:21:55,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/40b05c6c646546778922da72237185e9 is 50, key is test_row_0/B:col10/1731118913686/Put/seqid=0 2024-11-09T02:21:55,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741974_1150 (size=12001) 2024-11-09T02:21:55,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118975860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:55,862 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_2' in 6ec486933158f26905c39a8004ccca04 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:21:55,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:55,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:55,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:55,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118975990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:55,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118975991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:55,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118975992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:55,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118975992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:55,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:55,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:55,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:56,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/40b05c6c646546778922da72237185e9 2024-11-09T02:21:56,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/e01908720f80419096678d4da7c428df is 50, key is test_row_0/C:col10/1731118913686/Put/seqid=0 2024-11-09T02:21:56,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741975_1151 (size=12001) 2024-11-09T02:21:56,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/e01908720f80419096678d4da7c428df 2024-11-09T02:21:56,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/3b146ba2d2fa47f1ac0d9f2f8fdc0ed5 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/3b146ba2d2fa47f1ac0d9f2f8fdc0ed5 2024-11-09T02:21:56,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/3b146ba2d2fa47f1ac0d9f2f8fdc0ed5, entries=200, sequenceid=120, filesize=38.6 K 2024-11-09T02:21:56,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/40b05c6c646546778922da72237185e9 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/40b05c6c646546778922da72237185e9 2024-11-09T02:21:56,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/40b05c6c646546778922da72237185e9, entries=150, sequenceid=120, filesize=11.7 K 2024-11-09T02:21:56,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/e01908720f80419096678d4da7c428df as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/e01908720f80419096678d4da7c428df 2024-11-09T02:21:56,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/e01908720f80419096678d4da7c428df, entries=150, sequenceid=120, filesize=11.7 K 2024-11-09T02:21:56,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 6ec486933158f26905c39a8004ccca04 in 1371ms, sequenceid=120, compaction requested=true 2024-11-09T02:21:56,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:21:56,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ec486933158f26905c39a8004ccca04:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:21:56,204 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T02:21:56,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:56,204 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T02:21:56,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ec486933158f26905c39a8004ccca04:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:21:56,205 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T02:21:56,205 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 6ec486933158f26905c39a8004ccca04/B is initiating minor compaction (all files) 2024-11-09T02:21:56,206 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ec486933158f26905c39a8004ccca04/B in TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:56,206 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/19ca8b3d2ca6438b93e9eac575dba010, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/341615a7240c47d39e160b5a600c4986, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a4908276de084821bc2dd66130139244, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/40b05c6c646546778922da72237185e9] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp, totalSize=47.0 K 2024-11-09T02:21:56,206 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149701 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T02:21:56,206 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): 6ec486933158f26905c39a8004ccca04/A is initiating minor compaction (all files) 2024-11-09T02:21:56,206 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ec486933158f26905c39a8004ccca04/A in TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:56,206 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/3417402b38db48fea85429e450420471, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/d3d9c81e80e94d3b94b5754abdbe6b72, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/de4c7e99259146519557c6a21cfc99b3, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/3b146ba2d2fa47f1ac0d9f2f8fdc0ed5] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp, totalSize=146.2 K 2024-11-09T02:21:56,206 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 19ca8b3d2ca6438b93e9eac575dba010, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731118911175 2024-11-09T02:21:56,206 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:56,206 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. files: [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/3417402b38db48fea85429e450420471, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/d3d9c81e80e94d3b94b5754abdbe6b72, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/de4c7e99259146519557c6a21cfc99b3, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/3b146ba2d2fa47f1ac0d9f2f8fdc0ed5] 2024-11-09T02:21:56,207 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 341615a7240c47d39e160b5a600c4986, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731118911856 2024-11-09T02:21:56,207 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3417402b38db48fea85429e450420471, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731118911175 2024-11-09T02:21:56,208 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3d9c81e80e94d3b94b5754abdbe6b72, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731118911850 2024-11-09T02:21:56,208 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting a4908276de084821bc2dd66130139244, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731118912503 2024-11-09T02:21:56,208 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting de4c7e99259146519557c6a21cfc99b3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731118912503 2024-11-09T02:21:56,208 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 40b05c6c646546778922da72237185e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731118913673 2024-11-09T02:21:56,208 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b146ba2d2fa47f1ac0d9f2f8fdc0ed5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731118913673 2024-11-09T02:21:56,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:56,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ec486933158f26905c39a8004ccca04:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:21:56,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:21:56,221 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:21:56,224 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411093a37387974a14dfbb876acc45f403349_6ec486933158f26905c39a8004ccca04 store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:21:56,225 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ec486933158f26905c39a8004ccca04#B#compaction#130 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:56,225 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/bf1d4376bf8846258f30e5c0bc9843fa is 50, key is test_row_0/B:col10/1731118913686/Put/seqid=0 2024-11-09T02:21:56,228 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411093a37387974a14dfbb876acc45f403349_6ec486933158f26905c39a8004ccca04, store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:21:56,229 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093a37387974a14dfbb876acc45f403349_6ec486933158f26905c39a8004ccca04 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:21:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T02:21:56,231 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:21:56,231 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:21:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-09T02:21:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T02:21:56,234 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:21:56,235 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:21:56,235 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:21:56,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741977_1153 (size=4469) 2024-11-09T02:21:56,264 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ec486933158f26905c39a8004ccca04#A#compaction#129 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:56,265 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/f76b4d8e272c4b0d810f07b9214c5c82 is 175, key is test_row_0/A:col10/1731118913686/Put/seqid=0 2024-11-09T02:21:56,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741976_1152 (size=12241) 2024-11-09T02:21:56,287 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/bf1d4376bf8846258f30e5c0bc9843fa as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/bf1d4376bf8846258f30e5c0bc9843fa 2024-11-09T02:21:56,296 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ec486933158f26905c39a8004ccca04/B of 6ec486933158f26905c39a8004ccca04 into bf1d4376bf8846258f30e5c0bc9843fa(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:56,296 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:21:56,296 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., storeName=6ec486933158f26905c39a8004ccca04/B, priority=12, startTime=1731118916204; duration=0sec 2024-11-09T02:21:56,296 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:21:56,296 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ec486933158f26905c39a8004ccca04:B 2024-11-09T02:21:56,296 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T02:21:56,298 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T02:21:56,298 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 6ec486933158f26905c39a8004ccca04/C is initiating minor compaction (all files) 2024-11-09T02:21:56,298 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ec486933158f26905c39a8004ccca04/C in TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:56,298 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/7da40e7ddea54667b36e4c84c64dd5a5, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/6c96e7e9e5cd4e1eb115d2f2231aec36, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/391676a9990e4c7581d87608c16d1848, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/e01908720f80419096678d4da7c428df] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp, totalSize=47.0 K 2024-11-09T02:21:56,299 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 7da40e7ddea54667b36e4c84c64dd5a5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731118911175 2024-11-09T02:21:56,299 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c96e7e9e5cd4e1eb115d2f2231aec36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731118911856 2024-11-09T02:21:56,299 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 391676a9990e4c7581d87608c16d1848, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731118912503 2024-11-09T02:21:56,301 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting e01908720f80419096678d4da7c428df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731118913673 2024-11-09T02:21:56,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741978_1154 (size=31195) 2024-11-09T02:21:56,328 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ec486933158f26905c39a8004ccca04#C#compaction#131 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:56,328 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/1871b50daa3c4796b78b3b3168e14984 is 50, key is test_row_0/C:col10/1731118913686/Put/seqid=0 2024-11-09T02:21:56,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T02:21:56,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741979_1155 (size=12241) 2024-11-09T02:21:56,355 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/1871b50daa3c4796b78b3b3168e14984 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/1871b50daa3c4796b78b3b3168e14984 2024-11-09T02:21:56,362 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ec486933158f26905c39a8004ccca04/C of 6ec486933158f26905c39a8004ccca04 into 1871b50daa3c4796b78b3b3168e14984(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:56,362 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:21:56,362 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., storeName=6ec486933158f26905c39a8004ccca04/C, priority=12, startTime=1731118916210; duration=0sec 2024-11-09T02:21:56,362 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:56,362 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ec486933158f26905c39a8004ccca04:C 2024-11-09T02:21:56,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-09T02:21:56,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:56,388 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 6ec486933158f26905c39a8004ccca04 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-09T02:21:56,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091806af680f8f41ee836e81db56b314f7_6ec486933158f26905c39a8004ccca04 is 50, key is test_row_0/A:col10/1731118914843/Put/seqid=0 2024-11-09T02:21:56,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741980_1156 (size=12204) 2024-11-09T02:21:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T02:21:56,716 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/f76b4d8e272c4b0d810f07b9214c5c82 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/f76b4d8e272c4b0d810f07b9214c5c82 2024-11-09T02:21:56,724 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ec486933158f26905c39a8004ccca04/A of 6ec486933158f26905c39a8004ccca04 into f76b4d8e272c4b0d810f07b9214c5c82(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:56,724 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:21:56,724 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., storeName=6ec486933158f26905c39a8004ccca04/A, priority=12, startTime=1731118916204; duration=0sec 2024-11-09T02:21:56,725 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:56,725 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ec486933158f26905c39a8004ccca04:A 2024-11-09T02:21:56,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:56,821 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091806af680f8f41ee836e81db56b314f7_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091806af680f8f41ee836e81db56b314f7_6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:56,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/ec1a6c58c983480287b59208d131ade2, store: [table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:21:56,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/ec1a6c58c983480287b59208d131ade2 is 175, key is test_row_0/A:col10/1731118914843/Put/seqid=0 2024-11-09T02:21:56,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741981_1157 (size=31005) 2024-11-09T02:21:56,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T02:21:57,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:57,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:21:57,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118977031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:57,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:57,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118977033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:57,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118977034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:57,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118977036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:57,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:57,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118977140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:57,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:57,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118977150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:57,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:57,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118977150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118977150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:57,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:57,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:57,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:57,235 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/ec1a6c58c983480287b59208d131ade2 2024-11-09T02:21:57,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/1940603241854816ba35a1c6ad78c23d is 50, key is test_row_0/B:col10/1731118914843/Put/seqid=0 2024-11-09T02:21:57,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741982_1158 (size=12051) 2024-11-09T02:21:57,264 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/1940603241854816ba35a1c6ad78c23d 2024-11-09T02:21:57,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/ebc9dff3a34640fe84dd071736636b92 is 50, key is test_row_0/C:col10/1731118914843/Put/seqid=0 2024-11-09T02:21:57,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741983_1159 (size=12051) 2024-11-09T02:21:57,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118977350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:57,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118977360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:57,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:57,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118977361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:57,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118977361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:57,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:57,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T02:21:57,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:57,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118977670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:57,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118977670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:57,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118977670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:57,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:57,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118977670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:57,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:57,682 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/ebc9dff3a34640fe84dd071736636b92 2024-11-09T02:21:57,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/ec1a6c58c983480287b59208d131ade2 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/ec1a6c58c983480287b59208d131ade2 2024-11-09T02:21:57,692 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/ec1a6c58c983480287b59208d131ade2, entries=150, sequenceid=132, filesize=30.3 K 2024-11-09T02:21:57,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/1940603241854816ba35a1c6ad78c23d as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/1940603241854816ba35a1c6ad78c23d 2024-11-09T02:21:57,698 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/1940603241854816ba35a1c6ad78c23d, entries=150, sequenceid=132, filesize=11.8 K 2024-11-09T02:21:57,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/ebc9dff3a34640fe84dd071736636b92 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/ebc9dff3a34640fe84dd071736636b92 2024-11-09T02:21:57,705 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/ebc9dff3a34640fe84dd071736636b92, entries=150, sequenceid=132, filesize=11.8 K 2024-11-09T02:21:57,706 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 6ec486933158f26905c39a8004ccca04 in 1318ms, sequenceid=132, compaction requested=false 2024-11-09T02:21:57,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:21:57,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:57,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-09T02:21:57,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-09T02:21:57,709 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-09T02:21:57,709 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4730 sec 2024-11-09T02:21:57,711 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.4790 sec 2024-11-09T02:21:57,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:57,882 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:21:57,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ec486933158f26905c39a8004ccca04 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-09T02:21:57,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109c21058c1f6964cecb03b36e07256ecd3_6ec486933158f26905c39a8004ccca04 is 50, key is test_row_0/A:col10/1731118917031/Put/seqid=0 2024-11-09T02:21:57,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741984_1160 (size=14794) 2024-11-09T02:21:57,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:57,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118977901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:57,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:57,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:58,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:58,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118978010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:58,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:58,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118978180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:58,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:58,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:58,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118978180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:58,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:58,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118978190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:58,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:58,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118978190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:58,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:58,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118978220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:58,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:58,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:58,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:58,294 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:58,297 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109c21058c1f6964cecb03b36e07256ecd3_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c21058c1f6964cecb03b36e07256ecd3_6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:58,299 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/b48dd14ba87b40ceaccb2e025d0517e4, store: [table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:21:58,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/b48dd14ba87b40ceaccb2e025d0517e4 is 175, key is test_row_0/A:col10/1731118917031/Put/seqid=0 2024-11-09T02:21:58,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741985_1161 (size=39749) 2024-11-09T02:21:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T02:21:58,380 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:21:58,380 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:21:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-09T02:21:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T02:21:58,382 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:21:58,383 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:21:58,383 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:21:58,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T02:21:58,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T02:21:58,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:58,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:58,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:21:58,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118978530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:58,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:58,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:58,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:58,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:58,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:58,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:58,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:58,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T02:21:58,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:58,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:21:58,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:58,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:58,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:58,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T02:21:58,705 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=160, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/b48dd14ba87b40ceaccb2e025d0517e4 2024-11-09T02:21:58,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/a4b96985338244dd95d0a22e2581949f is 50, key is test_row_0/B:col10/1731118917031/Put/seqid=0 2024-11-09T02:21:58,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741986_1162 (size=12151) 2024-11-09T02:21:58,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T02:21:58,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:58,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:21:58,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:58,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:58,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:58,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T02:21:58,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:58,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:21:58,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:58,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:58,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:58,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:59,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T02:21:59,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118979040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:59,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/a4b96985338244dd95d0a22e2581949f 2024-11-09T02:21:59,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/62db74bccb4445209ec5b8f9271ef1af is 50, key is test_row_0/C:col10/1731118917031/Put/seqid=0 2024-11-09T02:21:59,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741987_1163 (size=12151) 2024-11-09T02:21:59,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/62db74bccb4445209ec5b8f9271ef1af 2024-11-09T02:21:59,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/b48dd14ba87b40ceaccb2e025d0517e4 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/b48dd14ba87b40ceaccb2e025d0517e4 2024-11-09T02:21:59,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T02:21:59,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:59,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:21:59,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:59,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:59,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:59,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:59,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/b48dd14ba87b40ceaccb2e025d0517e4, entries=200, sequenceid=160, filesize=38.8 K 2024-11-09T02:21:59,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/a4b96985338244dd95d0a22e2581949f as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a4b96985338244dd95d0a22e2581949f 2024-11-09T02:21:59,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a4b96985338244dd95d0a22e2581949f, entries=150, sequenceid=160, filesize=11.9 K 2024-11-09T02:21:59,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/62db74bccb4445209ec5b8f9271ef1af as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/62db74bccb4445209ec5b8f9271ef1af 2024-11-09T02:21:59,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/62db74bccb4445209ec5b8f9271ef1af, entries=150, sequenceid=160, filesize=11.9 K 2024-11-09T02:21:59,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 6ec486933158f26905c39a8004ccca04 in 1290ms, sequenceid=160, compaction requested=true 2024-11-09T02:21:59,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:21:59,172 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:59,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ec486933158f26905c39a8004ccca04:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:21:59,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:59,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,173 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:59,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ec486933158f26905c39a8004ccca04:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:21:59,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:59,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ec486933158f26905c39a8004ccca04:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:21:59,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:21:59,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,174 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101949 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:59,174 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): 6ec486933158f26905c39a8004ccca04/A is initiating minor compaction (all files) 2024-11-09T02:21:59,174 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ec486933158f26905c39a8004ccca04/A in TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:59,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,174 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/f76b4d8e272c4b0d810f07b9214c5c82, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/ec1a6c58c983480287b59208d131ade2, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/b48dd14ba87b40ceaccb2e025d0517e4] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp, totalSize=99.6 K 2024-11-09T02:21:59,174 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:59,175 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. files: [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/f76b4d8e272c4b0d810f07b9214c5c82, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/ec1a6c58c983480287b59208d131ade2, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/b48dd14ba87b40ceaccb2e025d0517e4] 2024-11-09T02:21:59,175 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:59,175 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 6ec486933158f26905c39a8004ccca04/B is initiating minor compaction (all files) 2024-11-09T02:21:59,175 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting f76b4d8e272c4b0d810f07b9214c5c82, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731118913673 2024-11-09T02:21:59,175 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ec486933158f26905c39a8004ccca04/B in TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:59,175 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/bf1d4376bf8846258f30e5c0bc9843fa, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/1940603241854816ba35a1c6ad78c23d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a4b96985338244dd95d0a22e2581949f] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp, totalSize=35.6 K 2024-11-09T02:21:59,176 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec1a6c58c983480287b59208d131ade2, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731118914836 2024-11-09T02:21:59,176 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting bf1d4376bf8846258f30e5c0bc9843fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731118913673 2024-11-09T02:21:59,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,176 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 1940603241854816ba35a1c6ad78c23d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731118914836 2024-11-09T02:21:59,176 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting b48dd14ba87b40ceaccb2e025d0517e4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731118917031 2024-11-09T02:21:59,176 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting a4b96985338244dd95d0a22e2581949f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731118917031 2024-11-09T02:21:59,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,184 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:21:59,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,190 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ec486933158f26905c39a8004ccca04#B#compaction#139 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:59,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,190 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/3b74eb81937e45efaceb910607100fd5 is 50, key is test_row_0/B:col10/1731118917031/Put/seqid=0 2024-11-09T02:21:59,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,192 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411091f231218524b40ddb28044150d232411_6ec486933158f26905c39a8004ccca04 store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:21:59,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,197 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411091f231218524b40ddb28044150d232411_6ec486933158f26905c39a8004ccca04, store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:21:59,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,197 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091f231218524b40ddb28044150d232411_6ec486933158f26905c39a8004ccca04 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:21:59,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:59,205 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:21:59,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ec486933158f26905c39a8004ccca04 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T02:21:59,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741989_1165 (size=4469) 2024-11-09T02:21:59,233 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ec486933158f26905c39a8004ccca04#A#compaction#138 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:59,234 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/3a1eeecd393449ff97b8f211c8aa09cc is 175, key is test_row_0/A:col10/1731118917031/Put/seqid=0 2024-11-09T02:21:59,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741988_1164 (size=12493) 2024-11-09T02:21:59,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109aed33e4f790547648fbffb1f22b3eff4_6ec486933158f26905c39a8004ccca04 is 50, key is test_row_0/A:col10/1731118919203/Put/seqid=0 2024-11-09T02:21:59,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741990_1166 (size=31447) 2024-11-09T02:21:59,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741991_1167 (size=14794) 2024-11-09T02:21:59,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:59,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118979270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:59,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:59,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118979271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:59,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:59,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118979272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:59,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:59,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118979273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:59,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T02:21:59,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:59,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:21:59,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:59,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:59,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:59,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118979390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118979390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118979390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:59,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:59,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118979390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:59,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:59,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:59,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T02:21:59,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:59,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:21:59,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:59,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:59,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T02:21:59,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:59,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118979600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:59,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:59,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:59,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118979600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:59,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:59,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118979600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:59,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:59,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:59,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118979600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:59,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:59,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:59,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T02:21:59,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:59,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:21:59,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:59,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:59,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:59,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:59,640 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/3b74eb81937e45efaceb910607100fd5 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/3b74eb81937e45efaceb910607100fd5 2024-11-09T02:21:59,646 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ec486933158f26905c39a8004ccca04/B of 6ec486933158f26905c39a8004ccca04 into 3b74eb81937e45efaceb910607100fd5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:59,646 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:21:59,646 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., storeName=6ec486933158f26905c39a8004ccca04/B, priority=13, startTime=1731118919173; duration=0sec 2024-11-09T02:21:59,646 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:21:59,646 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ec486933158f26905c39a8004ccca04:B 2024-11-09T02:21:59,646 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:21:59,647 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:21:59,647 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 6ec486933158f26905c39a8004ccca04/C is initiating minor compaction (all files) 2024-11-09T02:21:59,647 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ec486933158f26905c39a8004ccca04/C in TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:59,648 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/1871b50daa3c4796b78b3b3168e14984, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/ebc9dff3a34640fe84dd071736636b92, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/62db74bccb4445209ec5b8f9271ef1af] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp, totalSize=35.6 K 2024-11-09T02:21:59,648 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 1871b50daa3c4796b78b3b3168e14984, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731118913673 2024-11-09T02:21:59,648 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting ebc9dff3a34640fe84dd071736636b92, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731118914836 2024-11-09T02:21:59,649 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 62db74bccb4445209ec5b8f9271ef1af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731118917031 2024-11-09T02:21:59,655 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ec486933158f26905c39a8004ccca04#C#compaction#141 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:21:59,656 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/c95334adfaad4942a82a98747f4dcc22 is 50, key is test_row_0/C:col10/1731118917031/Put/seqid=0 2024-11-09T02:21:59,658 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/3a1eeecd393449ff97b8f211c8aa09cc as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/3a1eeecd393449ff97b8f211c8aa09cc 2024-11-09T02:21:59,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741992_1168 (size=12493) 2024-11-09T02:21:59,673 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/c95334adfaad4942a82a98747f4dcc22 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/c95334adfaad4942a82a98747f4dcc22 2024-11-09T02:21:59,674 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ec486933158f26905c39a8004ccca04/A of 6ec486933158f26905c39a8004ccca04 into 3a1eeecd393449ff97b8f211c8aa09cc(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:59,674 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:21:59,674 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., storeName=6ec486933158f26905c39a8004ccca04/A, priority=13, startTime=1731118919172; duration=0sec 2024-11-09T02:21:59,674 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:59,675 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:59,675 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ec486933158f26905c39a8004ccca04:A 2024-11-09T02:21:59,681 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ec486933158f26905c39a8004ccca04/C of 6ec486933158f26905c39a8004ccca04 into c95334adfaad4942a82a98747f4dcc22(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:21:59,681 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:21:59,681 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., storeName=6ec486933158f26905c39a8004ccca04/C, priority=13, startTime=1731118919173; duration=0sec 2024-11-09T02:21:59,681 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:21:59,681 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ec486933158f26905c39a8004ccca04:C 2024-11-09T02:21:59,681 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109aed33e4f790547648fbffb1f22b3eff4_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109aed33e4f790547648fbffb1f22b3eff4_6ec486933158f26905c39a8004ccca04 2024-11-09T02:21:59,683 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/022c60ae221b47959fd2fd6876abb34c, store: [table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:21:59,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/022c60ae221b47959fd2fd6876abb34c is 175, key is test_row_0/A:col10/1731118919203/Put/seqid=0 2024-11-09T02:21:59,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741993_1169 (size=39745) 2024-11-09T02:21:59,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T02:21:59,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:59,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:21:59,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:59,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:59,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:59,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:59,878 INFO [master/53f286d3af3b:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-09T02:21:59,878 INFO [master/53f286d3af3b:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-09T02:21:59,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:59,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118979910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:59,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:59,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118979910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:59,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:59,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:59,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118979910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:59,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:21:59,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118979910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:21:59,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:21:59,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:21:59,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T02:21:59,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:59,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:21:59,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:21:59,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:59,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:59,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:00,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:00,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118980060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:00,062 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_2' in 6ec486933158f26905c39a8004ccca04 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:22:00,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:00,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:00,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:00,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T02:22:00,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:00,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:22:00,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:00,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:00,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:00,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:00,093 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/022c60ae221b47959fd2fd6876abb34c 2024-11-09T02:22:00,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/f98df9a7e48748ae91b98c780f45ec8e is 50, key is test_row_0/B:col10/1731118919203/Put/seqid=0 2024-11-09T02:22:00,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741994_1170 (size=9757) 2024-11-09T02:22:00,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/f98df9a7e48748ae91b98c780f45ec8e 2024-11-09T02:22:00,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/4f0d323f80674378a0bc841cfcf2e50a is 50, key is test_row_0/C:col10/1731118919203/Put/seqid=0 2024-11-09T02:22:00,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741995_1171 (size=9757) 2024-11-09T02:22:00,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T02:22:00,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:00,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:22:00,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:00,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:00,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:00,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:00,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T02:22:00,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:00,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:22:00,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:00,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:00,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:00,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118980420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:00,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:00,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:00,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:00,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118980420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:00,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:00,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:00,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:00,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118980420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:00,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:00,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:00,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:00,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:00,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:00,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118980420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:00,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:00,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:00,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:00,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T02:22:00,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:00,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:22:00,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:00,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:00,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:00,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T02:22:00,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/4f0d323f80674378a0bc841cfcf2e50a 2024-11-09T02:22:00,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/022c60ae221b47959fd2fd6876abb34c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/022c60ae221b47959fd2fd6876abb34c 2024-11-09T02:22:00,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/022c60ae221b47959fd2fd6876abb34c, entries=200, sequenceid=171, filesize=38.8 K 2024-11-09T02:22:00,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/f98df9a7e48748ae91b98c780f45ec8e as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/f98df9a7e48748ae91b98c780f45ec8e 2024-11-09T02:22:00,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/f98df9a7e48748ae91b98c780f45ec8e, entries=100, sequenceid=171, filesize=9.5 K 2024-11-09T02:22:00,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/4f0d323f80674378a0bc841cfcf2e50a as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/4f0d323f80674378a0bc841cfcf2e50a 2024-11-09T02:22:00,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/4f0d323f80674378a0bc841cfcf2e50a, entries=100, sequenceid=171, filesize=9.5 K 2024-11-09T02:22:00,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 6ec486933158f26905c39a8004ccca04 in 1364ms, sequenceid=171, compaction requested=false 2024-11-09T02:22:00,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:22:00,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T02:22:00,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:00,681 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 6ec486933158f26905c39a8004ccca04 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T02:22:00,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411098a9fa6db1c9a42729c00c0173fbd1c78_6ec486933158f26905c39a8004ccca04 is 50, key is test_row_0/A:col10/1731118919264/Put/seqid=0 2024-11-09T02:22:00,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741996_1172 (size=12304) 2024-11-09T02:22:01,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:22:01,131 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411098a9fa6db1c9a42729c00c0173fbd1c78_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411098a9fa6db1c9a42729c00c0173fbd1c78_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:01,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/5547e456856542ee8d5ed9e805f589a9, store: [table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:01,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/5547e456856542ee8d5ed9e805f589a9 is 175, key is test_row_0/A:col10/1731118919264/Put/seqid=0 2024-11-09T02:22:01,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741997_1173 (size=31105) 2024-11-09T02:22:01,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:01,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:22:01,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118981440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:01,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118981440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:01,442 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_0' in 6ec486933158f26905c39a8004ccca04 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:22:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:01,443 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_2' in 6ec486933158f26905c39a8004ccca04 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:22:01,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118981441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:01,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118981442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:01,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118981549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:01,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:01,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118981549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:01,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:01,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:01,607 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/5547e456856542ee8d5ed9e805f589a9 2024-11-09T02:22:01,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/425371313f2f42f3bf693b9b5599c62e is 50, key is test_row_0/B:col10/1731118919264/Put/seqid=0 2024-11-09T02:22:01,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741998_1174 (size=12151) 2024-11-09T02:22:01,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:01,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118981760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:01,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:01,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:01,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118981760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:01,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:01,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:01,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:01,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:01,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:02,019 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/425371313f2f42f3bf693b9b5599c62e 2024-11-09T02:22:02,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/1fed847ea39246e6908887f0a182e8d2 is 50, key is test_row_0/C:col10/1731118919264/Put/seqid=0 2024-11-09T02:22:02,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741999_1175 (size=12151) 2024-11-09T02:22:02,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118982069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:02,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118982070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:02,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118982070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:02,070 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_2' in 6ec486933158f26905c39a8004ccca04 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:22:02,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:02,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:02,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:02,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:02,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:02,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:02,431 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/1fed847ea39246e6908887f0a182e8d2 2024-11-09T02:22:02,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/5547e456856542ee8d5ed9e805f589a9 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/5547e456856542ee8d5ed9e805f589a9 2024-11-09T02:22:02,440 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/5547e456856542ee8d5ed9e805f589a9, entries=150, sequenceid=199, filesize=30.4 K 2024-11-09T02:22:02,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/425371313f2f42f3bf693b9b5599c62e as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/425371313f2f42f3bf693b9b5599c62e 2024-11-09T02:22:02,446 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/425371313f2f42f3bf693b9b5599c62e, entries=150, sequenceid=199, filesize=11.9 K 2024-11-09T02:22:02,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/1fed847ea39246e6908887f0a182e8d2 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/1fed847ea39246e6908887f0a182e8d2 2024-11-09T02:22:02,455 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/1fed847ea39246e6908887f0a182e8d2, entries=150, sequenceid=199, filesize=11.9 K 2024-11-09T02:22:02,456 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 6ec486933158f26905c39a8004ccca04 in 1775ms, sequenceid=199, compaction requested=true 2024-11-09T02:22:02,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:22:02,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:02,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-09T02:22:02,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-09T02:22:02,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-09T02:22:02,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.0740 sec 2024-11-09T02:22:02,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 4.0800 sec 2024-11-09T02:22:02,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T02:22:02,550 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:22:02,550 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:22:02,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-09T02:22:02,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T02:22:02,552 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:22:02,553 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:22:02,553 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:22:02,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:02,583 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:22:02,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ec486933158f26905c39a8004ccca04 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T02:22:02,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f0bb6a2623054debb5a19c7cf371629c_6ec486933158f26905c39a8004ccca04 is 50, key is test_row_0/A:col10/1731118922582/Put/seqid=0 2024-11-09T02:22:02,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742000_1176 (size=14794) 2024-11-09T02:22:02,599 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:22:02,605 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f0bb6a2623054debb5a19c7cf371629c_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f0bb6a2623054debb5a19c7cf371629c_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:02,606 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/e97de2f34bed44638a5adf888fa73aa4, store: [table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:02,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/e97de2f34bed44638a5adf888fa73aa4 is 175, key is test_row_0/A:col10/1731118922582/Put/seqid=0 2024-11-09T02:22:02,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742001_1177 (size=39749) 2024-11-09T02:22:02,626 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/e97de2f34bed44638a5adf888fa73aa4 2024-11-09T02:22:02,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/166bfb09b8ce48bc8ea17f6c971383f8 is 50, key is test_row_0/B:col10/1731118922582/Put/seqid=0 2024-11-09T02:22:02,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:02,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118982630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:02,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:02,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118982636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:02,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:02,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:02,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:02,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742002_1178 (size=12151) 2024-11-09T02:22:02,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T02:22:02,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T02:22:02,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:02,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:22:02,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:02,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:02,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:02,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:02,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118982749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:02,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:02,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:02,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118982750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:02,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:02,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:02,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:02,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:02,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:02,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T02:22:02,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:02,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:22:02,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:02,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:02,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:02,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:02,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T02:22:02,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118982959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:02,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118982960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:02,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:02,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:02,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:02,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:02,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:02,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:03,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T02:22:03,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:03,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:22:03,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:03,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:03,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:03,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/166bfb09b8ce48bc8ea17f6c971383f8 2024-11-09T02:22:03,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/d081089af75b48228276268eb7685c83 is 50, key is test_row_0/C:col10/1731118922582/Put/seqid=0 2024-11-09T02:22:03,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742003_1179 (size=12151) 2024-11-09T02:22:03,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/d081089af75b48228276268eb7685c83 2024-11-09T02:22:03,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/e97de2f34bed44638a5adf888fa73aa4 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/e97de2f34bed44638a5adf888fa73aa4 2024-11-09T02:22:03,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/e97de2f34bed44638a5adf888fa73aa4, entries=200, sequenceid=211, filesize=38.8 K 2024-11-09T02:22:03,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/166bfb09b8ce48bc8ea17f6c971383f8 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/166bfb09b8ce48bc8ea17f6c971383f8 2024-11-09T02:22:03,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/166bfb09b8ce48bc8ea17f6c971383f8, entries=150, sequenceid=211, filesize=11.9 K 2024-11-09T02:22:03,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/d081089af75b48228276268eb7685c83 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/d081089af75b48228276268eb7685c83 2024-11-09T02:22:03,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/d081089af75b48228276268eb7685c83, entries=150, sequenceid=211, filesize=11.9 K 2024-11-09T02:22:03,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6ec486933158f26905c39a8004ccca04 in 497ms, sequenceid=211, compaction requested=true 2024-11-09T02:22:03,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:22:03,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ec486933158f26905c39a8004ccca04:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:22:03,080 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T02:22:03,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:03,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ec486933158f26905c39a8004ccca04:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:22:03,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:03,080 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T02:22:03,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ec486933158f26905c39a8004ccca04:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:22:03,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:03,082 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142046 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T02:22:03,082 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): 6ec486933158f26905c39a8004ccca04/A is initiating minor compaction (all files) 2024-11-09T02:22:03,082 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ec486933158f26905c39a8004ccca04/A in TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:03,082 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/3a1eeecd393449ff97b8f211c8aa09cc, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/022c60ae221b47959fd2fd6876abb34c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/5547e456856542ee8d5ed9e805f589a9, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/e97de2f34bed44638a5adf888fa73aa4] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp, totalSize=138.7 K 2024-11-09T02:22:03,082 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:03,082 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. files: [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/3a1eeecd393449ff97b8f211c8aa09cc, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/022c60ae221b47959fd2fd6876abb34c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/5547e456856542ee8d5ed9e805f589a9, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/e97de2f34bed44638a5adf888fa73aa4] 2024-11-09T02:22:03,085 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a1eeecd393449ff97b8f211c8aa09cc, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731118917031 2024-11-09T02:22:03,085 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T02:22:03,085 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 6ec486933158f26905c39a8004ccca04/B is initiating minor compaction (all files) 2024-11-09T02:22:03,085 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ec486933158f26905c39a8004ccca04/B in TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:03,085 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/3b74eb81937e45efaceb910607100fd5, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/f98df9a7e48748ae91b98c780f45ec8e, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/425371313f2f42f3bf693b9b5599c62e, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/166bfb09b8ce48bc8ea17f6c971383f8] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp, totalSize=45.5 K 2024-11-09T02:22:03,086 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b74eb81937e45efaceb910607100fd5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731118917031 2024-11-09T02:22:03,086 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 022c60ae221b47959fd2fd6876abb34c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731118917898 2024-11-09T02:22:03,086 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting f98df9a7e48748ae91b98c780f45ec8e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731118919195 2024-11-09T02:22:03,086 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5547e456856542ee8d5ed9e805f589a9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731118919264 2024-11-09T02:22:03,087 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 425371313f2f42f3bf693b9b5599c62e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731118919264 2024-11-09T02:22:03,087 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting e97de2f34bed44638a5adf888fa73aa4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731118921437 2024-11-09T02:22:03,087 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 166bfb09b8ce48bc8ea17f6c971383f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731118921437 2024-11-09T02:22:03,095 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:03,099 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ec486933158f26905c39a8004ccca04#B#compaction#151 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:03,099 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/589e0ed6cbe54f70b3aea3eb9cedb69b is 50, key is test_row_0/B:col10/1731118922582/Put/seqid=0 2024-11-09T02:22:03,105 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109f82fb0a721804ef6afb282821c1fc753_6ec486933158f26905c39a8004ccca04 store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:03,107 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109f82fb0a721804ef6afb282821c1fc753_6ec486933158f26905c39a8004ccca04, store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:03,108 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f82fb0a721804ef6afb282821c1fc753_6ec486933158f26905c39a8004ccca04 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:03,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742004_1180 (size=12629) 2024-11-09T02:22:03,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742005_1181 (size=4469) 2024-11-09T02:22:03,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T02:22:03,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:03,164 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 6ec486933158f26905c39a8004ccca04 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-09T02:22:03,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f6da6745897641d9a4ea412e0f5c500d_6ec486933158f26905c39a8004ccca04 is 50, key is test_row_0/A:col10/1731118922623/Put/seqid=0 2024-11-09T02:22:03,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742006_1182 (size=12304) 2024-11-09T02:22:03,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:22:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T02:22:03,186 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f6da6745897641d9a4ea412e0f5c500d_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f6da6745897641d9a4ea412e0f5c500d_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:03,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/042ae56d26f24141b73d8ecca6d3c1de, store: [table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:03,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/042ae56d26f24141b73d8ecca6d3c1de is 175, key is test_row_0/A:col10/1731118922623/Put/seqid=0 2024-11-09T02:22:03,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742007_1183 (size=31105) 2024-11-09T02:22:03,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:03,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:22:03,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118983288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:03,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:03,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:03,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:03,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118983289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:03,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:03,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:03,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:03,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118983400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:03,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118983400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:03,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118983460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:03,462 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_2' in 6ec486933158f26905c39a8004ccca04 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:22:03,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:03,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:03,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:03,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118983470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:03,473 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_0' in 6ec486933158f26905c39a8004ccca04 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:22:03,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:03,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:03,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:03,527 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/589e0ed6cbe54f70b3aea3eb9cedb69b as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/589e0ed6cbe54f70b3aea3eb9cedb69b 2024-11-09T02:22:03,531 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ec486933158f26905c39a8004ccca04#A#compaction#150 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:03,531 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/0ed38f6b96f6450995489883ef2cd3ff is 175, key is test_row_0/A:col10/1731118922582/Put/seqid=0 2024-11-09T02:22:03,533 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ec486933158f26905c39a8004ccca04/B of 6ec486933158f26905c39a8004ccca04 into 589e0ed6cbe54f70b3aea3eb9cedb69b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:03,533 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:22:03,533 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., storeName=6ec486933158f26905c39a8004ccca04/B, priority=12, startTime=1731118923080; duration=0sec 2024-11-09T02:22:03,533 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:03,533 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ec486933158f26905c39a8004ccca04:B 2024-11-09T02:22:03,533 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T02:22:03,535 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T02:22:03,535 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 6ec486933158f26905c39a8004ccca04/C is initiating minor compaction (all files) 2024-11-09T02:22:03,535 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ec486933158f26905c39a8004ccca04/C in TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:03,535 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/c95334adfaad4942a82a98747f4dcc22, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/4f0d323f80674378a0bc841cfcf2e50a, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/1fed847ea39246e6908887f0a182e8d2, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/d081089af75b48228276268eb7685c83] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp, totalSize=45.5 K 2024-11-09T02:22:03,535 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting c95334adfaad4942a82a98747f4dcc22, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731118917031 2024-11-09T02:22:03,536 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f0d323f80674378a0bc841cfcf2e50a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731118919195 2024-11-09T02:22:03,536 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fed847ea39246e6908887f0a182e8d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731118919264 2024-11-09T02:22:03,536 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting d081089af75b48228276268eb7685c83, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731118921437 2024-11-09T02:22:03,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742008_1184 (size=31583) 2024-11-09T02:22:03,546 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ec486933158f26905c39a8004ccca04#C#compaction#153 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:03,547 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/284e02e9227c434b881724c3dc8b4135 is 50, key is test_row_0/C:col10/1731118922582/Put/seqid=0 2024-11-09T02:22:03,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742009_1185 (size=12629) 2024-11-09T02:22:03,609 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/042ae56d26f24141b73d8ecca6d3c1de 2024-11-09T02:22:03,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118983610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:03,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118983611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:03,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:03,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:03,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:03,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:03,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:03,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:03,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/6d9fe293b9b64ff7a6354bca541016c4 is 50, key is test_row_0/B:col10/1731118922623/Put/seqid=0 2024-11-09T02:22:03,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742010_1186 (size=12151) 2024-11-09T02:22:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T02:22:03,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:03,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118983920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:03,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:03,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:03,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:03,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:03,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118983920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:03,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:03,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:03,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:03,949 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/0ed38f6b96f6450995489883ef2cd3ff as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/0ed38f6b96f6450995489883ef2cd3ff 2024-11-09T02:22:03,953 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ec486933158f26905c39a8004ccca04/A of 6ec486933158f26905c39a8004ccca04 into 0ed38f6b96f6450995489883ef2cd3ff(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:03,953 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:22:03,953 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., storeName=6ec486933158f26905c39a8004ccca04/A, priority=12, startTime=1731118923080; duration=0sec 2024-11-09T02:22:03,954 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:03,954 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ec486933158f26905c39a8004ccca04:A 2024-11-09T02:22:03,961 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/284e02e9227c434b881724c3dc8b4135 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/284e02e9227c434b881724c3dc8b4135 2024-11-09T02:22:03,967 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ec486933158f26905c39a8004ccca04/C of 6ec486933158f26905c39a8004ccca04 into 284e02e9227c434b881724c3dc8b4135(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:03,967 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:22:03,967 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., storeName=6ec486933158f26905c39a8004ccca04/C, priority=12, startTime=1731118923080; duration=0sec 2024-11-09T02:22:03,967 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:03,967 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ec486933158f26905c39a8004ccca04:C 2024-11-09T02:22:04,035 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/6d9fe293b9b64ff7a6354bca541016c4 2024-11-09T02:22:04,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/662e1ff85b7942afb7c2a7ee3f59d777 is 50, key is test_row_0/C:col10/1731118922623/Put/seqid=0 2024-11-09T02:22:04,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742011_1187 (size=12151) 2024-11-09T02:22:04,055 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/662e1ff85b7942afb7c2a7ee3f59d777 2024-11-09T02:22:04,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/042ae56d26f24141b73d8ecca6d3c1de as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/042ae56d26f24141b73d8ecca6d3c1de 2024-11-09T02:22:04,063 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/042ae56d26f24141b73d8ecca6d3c1de, entries=150, sequenceid=235, filesize=30.4 K 2024-11-09T02:22:04,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/6d9fe293b9b64ff7a6354bca541016c4 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/6d9fe293b9b64ff7a6354bca541016c4 2024-11-09T02:22:04,069 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/6d9fe293b9b64ff7a6354bca541016c4, entries=150, sequenceid=235, filesize=11.9 K 2024-11-09T02:22:04,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/662e1ff85b7942afb7c2a7ee3f59d777 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/662e1ff85b7942afb7c2a7ee3f59d777 2024-11-09T02:22:04,075 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/662e1ff85b7942afb7c2a7ee3f59d777, entries=150, sequenceid=235, filesize=11.9 K 2024-11-09T02:22:04,076 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 6ec486933158f26905c39a8004ccca04 in 912ms, sequenceid=235, compaction requested=false 2024-11-09T02:22:04,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:22:04,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:04,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-09T02:22:04,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-09T02:22:04,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-09T02:22:04,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5240 sec 2024-11-09T02:22:04,081 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.5300 sec 2024-11-09T02:22:04,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:04,432 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:22:04,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ec486933158f26905c39a8004ccca04 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-09T02:22:04,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110985371048506841d492b091e8e06821ce_6ec486933158f26905c39a8004ccca04 is 50, key is test_row_0/A:col10/1731118924432/Put/seqid=0 2024-11-09T02:22:04,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742012_1188 (size=17284) 2024-11-09T02:22:04,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:04,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118984474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:04,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:04,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:04,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118984474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:04,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:04,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:04,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:04,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:04,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:04,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:04,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118984589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:04,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:04,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118984589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:04,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T02:22:04,710 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:22:04,710 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:22:04,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-09T02:22:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T02:22:04,712 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:22:04,712 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:22:04,712 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:22:04,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:04,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118984799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:04,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:04,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118984799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:04,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:04,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:04,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:04,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:04,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:04,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:04,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T02:22:04,855 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:22:04,859 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110985371048506841d492b091e8e06821ce_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110985371048506841d492b091e8e06821ce_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:04,860 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/4593cb74158e4801948478d15e1c91d0, store: [table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:04,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/4593cb74158e4801948478d15e1c91d0 is 175, key is test_row_0/A:col10/1731118924432/Put/seqid=0 2024-11-09T02:22:04,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-09T02:22:04,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:04,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:22:04,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:04,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:04,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:04,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:04,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742013_1189 (size=48389) 2024-11-09T02:22:04,870 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/4593cb74158e4801948478d15e1c91d0 2024-11-09T02:22:04,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/17172f5939a24f18bba6f022fe0a42c5 is 50, key is test_row_0/B:col10/1731118924432/Put/seqid=0 2024-11-09T02:22:04,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742014_1190 (size=12151) 2024-11-09T02:22:05,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-09T02:22:05,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:05,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:22:05,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:05,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:05,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:05,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:05,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T02:22:05,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:05,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118985110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:05,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:05,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:05,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:05,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118985110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:05,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-09T02:22:05,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:05,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:22:05,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:05,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:05,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:05,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/17172f5939a24f18bba6f022fe0a42c5 2024-11-09T02:22:05,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/8f9eef52699e490c891e1ec73380253d is 50, key is test_row_0/C:col10/1731118924432/Put/seqid=0 2024-11-09T02:22:05,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742015_1191 (size=12151) 2024-11-09T02:22:05,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-09T02:22:05,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:05,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:22:05,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:05,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:05,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:05,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T02:22:05,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-09T02:22:05,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:05,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:22:05,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:05,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:05,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:05,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:05,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118985620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:05,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118985620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:05,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-09T02:22:05,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:05,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:22:05,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:05,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:05,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:05,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/8f9eef52699e490c891e1ec73380253d 2024-11-09T02:22:05,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/4593cb74158e4801948478d15e1c91d0 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/4593cb74158e4801948478d15e1c91d0 2024-11-09T02:22:05,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/4593cb74158e4801948478d15e1c91d0, entries=250, sequenceid=252, filesize=47.3 K 2024-11-09T02:22:05,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/17172f5939a24f18bba6f022fe0a42c5 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/17172f5939a24f18bba6f022fe0a42c5 2024-11-09T02:22:05,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/17172f5939a24f18bba6f022fe0a42c5, entries=150, sequenceid=252, filesize=11.9 K 2024-11-09T02:22:05,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/8f9eef52699e490c891e1ec73380253d as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/8f9eef52699e490c891e1ec73380253d 2024-11-09T02:22:05,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/8f9eef52699e490c891e1ec73380253d, entries=150, sequenceid=252, filesize=11.9 K 2024-11-09T02:22:05,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 6ec486933158f26905c39a8004ccca04 in 1290ms, sequenceid=252, compaction requested=true 2024-11-09T02:22:05,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:22:05,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ec486933158f26905c39a8004ccca04:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:22:05,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:05,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ec486933158f26905c39a8004ccca04:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:22:05,723 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:05,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:05,723 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:05,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ec486933158f26905c39a8004ccca04:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:22:05,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:05,724 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:05,724 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111077 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:05,724 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 6ec486933158f26905c39a8004ccca04/B is initiating minor compaction (all files) 2024-11-09T02:22:05,724 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): 6ec486933158f26905c39a8004ccca04/A is initiating minor compaction (all files) 2024-11-09T02:22:05,724 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ec486933158f26905c39a8004ccca04/A in TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:05,724 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ec486933158f26905c39a8004ccca04/B in TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:05,724 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/589e0ed6cbe54f70b3aea3eb9cedb69b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/6d9fe293b9b64ff7a6354bca541016c4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/17172f5939a24f18bba6f022fe0a42c5] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp, totalSize=36.1 K 2024-11-09T02:22:05,724 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/0ed38f6b96f6450995489883ef2cd3ff, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/042ae56d26f24141b73d8ecca6d3c1de, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/4593cb74158e4801948478d15e1c91d0] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp, totalSize=108.5 K 2024-11-09T02:22:05,724 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:05,724 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. files: [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/0ed38f6b96f6450995489883ef2cd3ff, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/042ae56d26f24141b73d8ecca6d3c1de, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/4593cb74158e4801948478d15e1c91d0] 2024-11-09T02:22:05,724 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 589e0ed6cbe54f70b3aea3eb9cedb69b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731118921437 2024-11-09T02:22:05,724 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ed38f6b96f6450995489883ef2cd3ff, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731118921437 2024-11-09T02:22:05,725 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 042ae56d26f24141b73d8ecca6d3c1de, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731118922623 2024-11-09T02:22:05,725 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d9fe293b9b64ff7a6354bca541016c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731118922623 2024-11-09T02:22:05,725 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4593cb74158e4801948478d15e1c91d0, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731118923283 2024-11-09T02:22:05,725 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 17172f5939a24f18bba6f022fe0a42c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731118923283 2024-11-09T02:22:05,730 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:05,732 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ec486933158f26905c39a8004ccca04#B#compaction#160 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:05,732 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110955f70297102f4defa0b91ff0cca07d8b_6ec486933158f26905c39a8004ccca04 store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:05,733 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/a1afb25d9af44f7a97d1c5591fc1f780 is 50, key is test_row_0/B:col10/1731118924432/Put/seqid=0 2024-11-09T02:22:05,734 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110955f70297102f4defa0b91ff0cca07d8b_6ec486933158f26905c39a8004ccca04, store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:05,734 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110955f70297102f4defa0b91ff0cca07d8b_6ec486933158f26905c39a8004ccca04 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:05,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742017_1193 (size=4469) 2024-11-09T02:22:05,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742016_1192 (size=12731) 2024-11-09T02:22:05,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-09T02:22:05,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:05,783 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 6ec486933158f26905c39a8004ccca04 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-09T02:22:05,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090599c2310caf4b198f391bdc9ca501f9_6ec486933158f26905c39a8004ccca04 is 50, key is test_row_0/A:col10/1731118924472/Put/seqid=0 2024-11-09T02:22:05,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742018_1194 (size=12454) 2024-11-09T02:22:05,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:22:05,801 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090599c2310caf4b198f391bdc9ca501f9_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090599c2310caf4b198f391bdc9ca501f9_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:05,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/a70320c2ed784b8f9fd605782831aab3, store: [table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:05,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/a70320c2ed784b8f9fd605782831aab3 is 175, key is test_row_0/A:col10/1731118924472/Put/seqid=0 2024-11-09T02:22:05,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742019_1195 (size=31255) 2024-11-09T02:22:05,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T02:22:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:06,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:22:06,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118986131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:06,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:06,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:06,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:06,143 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ec486933158f26905c39a8004ccca04#A#compaction#159 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:06,143 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/7f367d756b474e31aa07e47ee294f60b is 175, key is test_row_0/A:col10/1731118924432/Put/seqid=0 2024-11-09T02:22:06,227 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/a70320c2ed784b8f9fd605782831aab3 2024-11-09T02:22:06,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742020_1196 (size=31685) 2024-11-09T02:22:06,232 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/a1afb25d9af44f7a97d1c5591fc1f780 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a1afb25d9af44f7a97d1c5591fc1f780 2024-11-09T02:22:06,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/101dca799d5b47078d3a1680e30dbbd1 is 50, key is test_row_0/B:col10/1731118924472/Put/seqid=0 2024-11-09T02:22:06,238 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ec486933158f26905c39a8004ccca04/B of 6ec486933158f26905c39a8004ccca04 into a1afb25d9af44f7a97d1c5591fc1f780(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:06,238 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:22:06,238 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., storeName=6ec486933158f26905c39a8004ccca04/B, priority=13, startTime=1731118925723; duration=0sec 2024-11-09T02:22:06,238 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:06,238 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ec486933158f26905c39a8004ccca04:B 2024-11-09T02:22:06,238 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:06,240 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:06,240 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 6ec486933158f26905c39a8004ccca04/C is initiating minor compaction (all files) 2024-11-09T02:22:06,241 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ec486933158f26905c39a8004ccca04/C in TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:06,241 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/284e02e9227c434b881724c3dc8b4135, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/662e1ff85b7942afb7c2a7ee3f59d777, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/8f9eef52699e490c891e1ec73380253d] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp, totalSize=36.1 K 2024-11-09T02:22:06,242 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 284e02e9227c434b881724c3dc8b4135, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731118921437 2024-11-09T02:22:06,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742021_1197 (size=12301) 2024-11-09T02:22:06,242 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 662e1ff85b7942afb7c2a7ee3f59d777, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731118922623 2024-11-09T02:22:06,242 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/101dca799d5b47078d3a1680e30dbbd1 2024-11-09T02:22:06,243 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f9eef52699e490c891e1ec73380253d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731118923283 2024-11-09T02:22:06,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118986241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:06,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:06,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:06,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:06,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/60ef84ccc5eb4018b238e52d406ea774 is 50, key is test_row_0/C:col10/1731118924472/Put/seqid=0 2024-11-09T02:22:06,253 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ec486933158f26905c39a8004ccca04#C#compaction#164 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:06,253 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/2d45d55253cf4a52926eb08d281d0f8d is 50, key is test_row_0/C:col10/1731118924432/Put/seqid=0 2024-11-09T02:22:06,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742022_1198 (size=12301) 2024-11-09T02:22:06,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742023_1199 (size=12731) 2024-11-09T02:22:06,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118986449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:06,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:06,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:06,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:06,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118986630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:06,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118986630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:06,634 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_0' in 6ec486933158f26905c39a8004ccca04 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:22:06,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:06,634 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_0' in 6ec486933158f26905c39a8004ccca04 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:22:06,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:06,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:06,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:06,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:06,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:06,636 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/7f367d756b474e31aa07e47ee294f60b as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/7f367d756b474e31aa07e47ee294f60b 2024-11-09T02:22:06,641 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ec486933158f26905c39a8004ccca04/A of 6ec486933158f26905c39a8004ccca04 into 7f367d756b474e31aa07e47ee294f60b(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:06,641 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:22:06,641 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., storeName=6ec486933158f26905c39a8004ccca04/A, priority=13, startTime=1731118925722; duration=0sec 2024-11-09T02:22:06,641 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:06,641 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ec486933158f26905c39a8004ccca04:A 2024-11-09T02:22:06,662 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/60ef84ccc5eb4018b238e52d406ea774 2024-11-09T02:22:06,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/a70320c2ed784b8f9fd605782831aab3 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/a70320c2ed784b8f9fd605782831aab3 2024-11-09T02:22:06,671 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/a70320c2ed784b8f9fd605782831aab3, entries=150, sequenceid=274, filesize=30.5 K 2024-11-09T02:22:06,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/101dca799d5b47078d3a1680e30dbbd1 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/101dca799d5b47078d3a1680e30dbbd1 2024-11-09T02:22:06,675 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/2d45d55253cf4a52926eb08d281d0f8d as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/2d45d55253cf4a52926eb08d281d0f8d 2024-11-09T02:22:06,677 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/101dca799d5b47078d3a1680e30dbbd1, entries=150, sequenceid=274, filesize=12.0 K 2024-11-09T02:22:06,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/60ef84ccc5eb4018b238e52d406ea774 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/60ef84ccc5eb4018b238e52d406ea774 2024-11-09T02:22:06,682 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ec486933158f26905c39a8004ccca04/C of 6ec486933158f26905c39a8004ccca04 into 2d45d55253cf4a52926eb08d281d0f8d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:06,682 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:22:06,682 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., storeName=6ec486933158f26905c39a8004ccca04/C, priority=13, startTime=1731118925723; duration=0sec 2024-11-09T02:22:06,682 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:06,682 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ec486933158f26905c39a8004ccca04:C 2024-11-09T02:22:06,686 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/60ef84ccc5eb4018b238e52d406ea774, entries=150, sequenceid=274, filesize=12.0 K 2024-11-09T02:22:06,687 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 6ec486933158f26905c39a8004ccca04 in 904ms, sequenceid=274, compaction requested=false 2024-11-09T02:22:06,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:22:06,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:06,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-09T02:22:06,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-09T02:22:06,690 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-09T02:22:06,690 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9770 sec 2024-11-09T02:22:06,692 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.9810 sec 2024-11-09T02:22:06,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:06,761 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:22:06,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ec486933158f26905c39a8004ccca04 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-09T02:22:06,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f3fc8e7fc9584cd7b45a295c55c8926d_6ec486933158f26905c39a8004ccca04 is 50, key is test_row_0/A:col10/1731118926105/Put/seqid=0 2024-11-09T02:22:06,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742024_1200 (size=14994) 2024-11-09T02:22:06,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:06,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118986820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:06,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:06,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:06,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:06,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T02:22:06,870 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:22:06,871 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:22:06,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-09T02:22:06,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T02:22:06,872 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:22:06,873 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:22:06,873 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:22:06,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118986929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:06,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:06,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:06,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T02:22:07,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-09T02:22:07,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:07,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:22:07,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:07,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:07,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:07,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:07,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118987140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:07,176 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:22:07,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-09T02:22:07,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:07,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:22:07,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:07,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:07,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:07,181 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f3fc8e7fc9584cd7b45a295c55c8926d_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f3fc8e7fc9584cd7b45a295c55c8926d_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:07,182 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/f0d475cae768481fabc26f74badd6926, store: [table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:07,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/f0d475cae768481fabc26f74badd6926 is 175, key is test_row_0/A:col10/1731118926105/Put/seqid=0 2024-11-09T02:22:07,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742025_1201 (size=39949) 2024-11-09T02:22:07,188 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/f0d475cae768481fabc26f74badd6926 2024-11-09T02:22:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T02:22:07,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/a692e5ba057946f1a71f0d2770793672 is 50, key is test_row_0/B:col10/1731118926105/Put/seqid=0 2024-11-09T02:22:07,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742026_1202 (size=12301) 2024-11-09T02:22:07,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/a692e5ba057946f1a71f0d2770793672 2024-11-09T02:22:07,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/8d2346897e4b4b0685633fe373c8cef1 is 50, key is test_row_0/C:col10/1731118926105/Put/seqid=0 2024-11-09T02:22:07,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742027_1203 (size=12301) 2024-11-09T02:22:07,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-09T02:22:07,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:07,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:22:07,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:07,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:07,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:07,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118987450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:07,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:07,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:07,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:07,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-09T02:22:07,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:07,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:22:07,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:07,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:07,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T02:22:07,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48428 deadline: 1731118987500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:07,503 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_2' in 6ec486933158f26905c39a8004ccca04 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:22:07,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:07,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:07,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:07,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:07,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48372 deadline: 1731118987510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:07,514 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_0' in 6ec486933158f26905c39a8004ccca04 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:22:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:07,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/8d2346897e4b4b0685633fe373c8cef1 2024-11-09T02:22:07,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/f0d475cae768481fabc26f74badd6926 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/f0d475cae768481fabc26f74badd6926 2024-11-09T02:22:07,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/f0d475cae768481fabc26f74badd6926, entries=200, sequenceid=292, filesize=39.0 K 2024-11-09T02:22:07,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/a692e5ba057946f1a71f0d2770793672 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a692e5ba057946f1a71f0d2770793672 2024-11-09T02:22:07,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a692e5ba057946f1a71f0d2770793672, entries=150, sequenceid=292, filesize=12.0 K 2024-11-09T02:22:07,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/8d2346897e4b4b0685633fe373c8cef1 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/8d2346897e4b4b0685633fe373c8cef1 2024-11-09T02:22:07,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/8d2346897e4b4b0685633fe373c8cef1, entries=150, sequenceid=292, filesize=12.0 K 2024-11-09T02:22:07,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 6ec486933158f26905c39a8004ccca04 in 872ms, sequenceid=292, compaction requested=true 2024-11-09T02:22:07,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:22:07,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ec486933158f26905c39a8004ccca04:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:22:07,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:07,634 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:07,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ec486933158f26905c39a8004ccca04:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:22:07,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:07,634 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:07,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ec486933158f26905c39a8004ccca04:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:22:07,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:07,635 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:07,635 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:07,635 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 6ec486933158f26905c39a8004ccca04/B is initiating minor compaction (all files) 2024-11-09T02:22:07,635 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ec486933158f26905c39a8004ccca04/B in TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:07,635 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): 6ec486933158f26905c39a8004ccca04/A is initiating minor compaction (all files) 2024-11-09T02:22:07,635 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a1afb25d9af44f7a97d1c5591fc1f780, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/101dca799d5b47078d3a1680e30dbbd1, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a692e5ba057946f1a71f0d2770793672] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp, totalSize=36.5 K 2024-11-09T02:22:07,635 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ec486933158f26905c39a8004ccca04/A in TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:07,635 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/7f367d756b474e31aa07e47ee294f60b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/a70320c2ed784b8f9fd605782831aab3, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/f0d475cae768481fabc26f74badd6926] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp, totalSize=100.5 K 2024-11-09T02:22:07,635 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:07,635 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. files: [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/7f367d756b474e31aa07e47ee294f60b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/a70320c2ed784b8f9fd605782831aab3, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/f0d475cae768481fabc26f74badd6926] 2024-11-09T02:22:07,636 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting a1afb25d9af44f7a97d1c5591fc1f780, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731118923283 2024-11-09T02:22:07,636 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f367d756b474e31aa07e47ee294f60b, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731118923283 2024-11-09T02:22:07,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-09T02:22:07,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:07,637 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 6ec486933158f26905c39a8004ccca04 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-09T02:22:07,637 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting a70320c2ed784b8f9fd605782831aab3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731118924457 2024-11-09T02:22:07,637 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 101dca799d5b47078d3a1680e30dbbd1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731118924457 2024-11-09T02:22:07,637 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0d475cae768481fabc26f74badd6926, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731118926105 2024-11-09T02:22:07,639 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting a692e5ba057946f1a71f0d2770793672, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731118926105 2024-11-09T02:22:07,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109118af24d0bb64c61ad6885cb00c1cc41_6ec486933158f26905c39a8004ccca04 is 50, key is test_row_0/A:col10/1731118926815/Put/seqid=0 2024-11-09T02:22:07,651 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:07,656 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ec486933158f26905c39a8004ccca04#B#compaction#170 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:07,656 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110936c7ed5564b844068154827378e6a750_6ec486933158f26905c39a8004ccca04 store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:07,656 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/aa88a0ff885e4f11b5f33fa58985c584 is 50, key is test_row_0/B:col10/1731118926105/Put/seqid=0 2024-11-09T02:22:07,659 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110936c7ed5564b844068154827378e6a750_6ec486933158f26905c39a8004ccca04, store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:07,659 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110936c7ed5564b844068154827378e6a750_6ec486933158f26905c39a8004ccca04 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:07,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742029_1205 (size=12983) 2024-11-09T02:22:07,670 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/aa88a0ff885e4f11b5f33fa58985c584 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/aa88a0ff885e4f11b5f33fa58985c584 2024-11-09T02:22:07,675 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ec486933158f26905c39a8004ccca04/B of 6ec486933158f26905c39a8004ccca04 into aa88a0ff885e4f11b5f33fa58985c584(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:07,675 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:22:07,675 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., storeName=6ec486933158f26905c39a8004ccca04/B, priority=13, startTime=1731118927634; duration=0sec 2024-11-09T02:22:07,675 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:07,675 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ec486933158f26905c39a8004ccca04:B 2024-11-09T02:22:07,675 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:07,676 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:07,676 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 6ec486933158f26905c39a8004ccca04/C is initiating minor compaction (all files) 2024-11-09T02:22:07,677 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ec486933158f26905c39a8004ccca04/C in TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:07,677 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/2d45d55253cf4a52926eb08d281d0f8d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/60ef84ccc5eb4018b238e52d406ea774, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/8d2346897e4b4b0685633fe373c8cef1] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp, totalSize=36.5 K 2024-11-09T02:22:07,677 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d45d55253cf4a52926eb08d281d0f8d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731118923283 2024-11-09T02:22:07,677 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 60ef84ccc5eb4018b238e52d406ea774, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731118924457 2024-11-09T02:22:07,677 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d2346897e4b4b0685633fe373c8cef1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731118926105 2024-11-09T02:22:07,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742030_1206 (size=4469) 2024-11-09T02:22:07,681 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ec486933158f26905c39a8004ccca04#A#compaction#169 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:07,682 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/4d12e269a59e4cbe8542435fd002511c is 175, key is test_row_0/A:col10/1731118926105/Put/seqid=0 2024-11-09T02:22:07,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742028_1204 (size=12454) 2024-11-09T02:22:07,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742031_1207 (size=31937) 2024-11-09T02:22:07,710 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/4d12e269a59e4cbe8542435fd002511c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/4d12e269a59e4cbe8542435fd002511c 2024-11-09T02:22:07,714 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ec486933158f26905c39a8004ccca04#C#compaction#171 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:07,715 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/0af0f3a784f64337bf20911899e6bbb7 is 50, key is test_row_0/C:col10/1731118926105/Put/seqid=0 2024-11-09T02:22:07,717 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ec486933158f26905c39a8004ccca04/A of 6ec486933158f26905c39a8004ccca04 into 4d12e269a59e4cbe8542435fd002511c(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:07,717 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:22:07,717 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., storeName=6ec486933158f26905c39a8004ccca04/A, priority=13, startTime=1731118927633; duration=0sec 2024-11-09T02:22:07,720 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:07,720 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ec486933158f26905c39a8004ccca04:A 2024-11-09T02:22:07,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742032_1208 (size=12983) 2024-11-09T02:22:07,746 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/0af0f3a784f64337bf20911899e6bbb7 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/0af0f3a784f64337bf20911899e6bbb7 2024-11-09T02:22:07,755 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ec486933158f26905c39a8004ccca04/C of 6ec486933158f26905c39a8004ccca04 into 0af0f3a784f64337bf20911899e6bbb7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:07,755 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:22:07,755 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., storeName=6ec486933158f26905c39a8004ccca04/C, priority=13, startTime=1731118927634; duration=0sec 2024-11-09T02:22:07,756 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:07,756 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ec486933158f26905c39a8004ccca04:C 2024-11-09T02:22:07,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:22:07,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:08,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:08,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118987998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:08,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T02:22:08,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:22:08,101 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109118af24d0bb64c61ad6885cb00c1cc41_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109118af24d0bb64c61ad6885cb00c1cc41_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:08,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/1e5de62dc8874b3e8ed39a764bcd3002, store: [table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:08,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/1e5de62dc8874b3e8ed39a764bcd3002 is 175, key is test_row_0/A:col10/1731118926815/Put/seqid=0 2024-11-09T02:22:08,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:08,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118988110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:08,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:08,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:08,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:08,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742033_1209 (size=31255) 2024-11-09T02:22:08,115 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/1e5de62dc8874b3e8ed39a764bcd3002 2024-11-09T02:22:08,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/11d99ad6695c4097b5d142a46e66d539 is 50, key is test_row_0/B:col10/1731118926815/Put/seqid=0 2024-11-09T02:22:08,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742034_1210 (size=12301) 2024-11-09T02:22:08,136 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/11d99ad6695c4097b5d142a46e66d539 2024-11-09T02:22:08,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/f9f53597c0c545968b25fbe6165e626c is 50, key is test_row_0/C:col10/1731118926815/Put/seqid=0 2024-11-09T02:22:08,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742035_1211 (size=12301) 2024-11-09T02:22:08,157 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/f9f53597c0c545968b25fbe6165e626c 2024-11-09T02:22:08,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/1e5de62dc8874b3e8ed39a764bcd3002 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/1e5de62dc8874b3e8ed39a764bcd3002 2024-11-09T02:22:08,171 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/1e5de62dc8874b3e8ed39a764bcd3002, entries=150, sequenceid=313, filesize=30.5 K 2024-11-09T02:22:08,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/11d99ad6695c4097b5d142a46e66d539 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/11d99ad6695c4097b5d142a46e66d539 2024-11-09T02:22:08,184 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/11d99ad6695c4097b5d142a46e66d539, entries=150, sequenceid=313, filesize=12.0 K 2024-11-09T02:22:08,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/f9f53597c0c545968b25fbe6165e626c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/f9f53597c0c545968b25fbe6165e626c 2024-11-09T02:22:08,190 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/f9f53597c0c545968b25fbe6165e626c, entries=150, sequenceid=313, filesize=12.0 K 2024-11-09T02:22:08,191 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 6ec486933158f26905c39a8004ccca04 in 554ms, sequenceid=313, compaction requested=false 2024-11-09T02:22:08,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:22:08,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:08,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-09T02:22:08,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-09T02:22:08,197 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-09T02:22:08,197 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3200 sec 2024-11-09T02:22:08,198 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.3260 sec 2024-11-09T02:22:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:08,322 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:22:08,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ec486933158f26905c39a8004ccca04 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-09T02:22:08,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109fb58ba38dc374ea796991e7a84d0164e_6ec486933158f26905c39a8004ccca04 is 50, key is test_row_0/A:col10/1731118927996/Put/seqid=0 2024-11-09T02:22:08,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742036_1212 (size=14994) 2024-11-09T02:22:08,349 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:22:08,354 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109fb58ba38dc374ea796991e7a84d0164e_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109fb58ba38dc374ea796991e7a84d0164e_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:08,355 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/e18530dad5ba4236b761b196b06a1b7f, store: [table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:08,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/e18530dad5ba4236b761b196b06a1b7f is 175, key is test_row_0/A:col10/1731118927996/Put/seqid=0 2024-11-09T02:22:08,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742037_1213 (size=39949) 2024-11-09T02:22:08,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:08,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118988381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:08,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:08,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:08,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:08,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:08,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118988489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:08,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:08,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:08,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:08,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:08,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48416 deadline: 1731118988639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:08,641 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_0' in 6ec486933158f26905c39a8004ccca04 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:22:08,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:08,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:08,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:08,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:08,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48430 deadline: 1731118988650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:08,651 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_0' in 6ec486933158f26905c39a8004ccca04 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:22:08,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:08,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:08,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:08,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:08,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118988700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:08,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:08,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:08,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:08,763 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=332, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/e18530dad5ba4236b761b196b06a1b7f 2024-11-09T02:22:08,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/782aa261591449da925f0b20aab61f27 is 50, key is test_row_0/B:col10/1731118927996/Put/seqid=0 2024-11-09T02:22:08,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742038_1214 (size=12301) 2024-11-09T02:22:09,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118989010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:09,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:09,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:09,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T02:22:09,020 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:22:09,020 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:22:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-09T02:22:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T02:22:09,022 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:22:09,022 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:22:09,023 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:22:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T02:22:09,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-09T02:22:09,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:09,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:22:09,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:09,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:09,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:09,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/782aa261591449da925f0b20aab61f27 2024-11-09T02:22:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:09,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/bd41381064d3405499e527dfde900460 is 50, key is test_row_0/C:col10/1731118927996/Put/seqid=0 2024-11-09T02:22:09,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742039_1215 (size=12301) 2024-11-09T02:22:09,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/bd41381064d3405499e527dfde900460 2024-11-09T02:22:09,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/e18530dad5ba4236b761b196b06a1b7f as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/e18530dad5ba4236b761b196b06a1b7f 2024-11-09T02:22:09,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/e18530dad5ba4236b761b196b06a1b7f, entries=200, sequenceid=332, filesize=39.0 K 2024-11-09T02:22:09,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/782aa261591449da925f0b20aab61f27 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/782aa261591449da925f0b20aab61f27 2024-11-09T02:22:09,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/782aa261591449da925f0b20aab61f27, entries=150, sequenceid=332, filesize=12.0 K 2024-11-09T02:22:09,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/bd41381064d3405499e527dfde900460 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/bd41381064d3405499e527dfde900460 2024-11-09T02:22:09,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/bd41381064d3405499e527dfde900460, entries=150, sequenceid=332, filesize=12.0 K 2024-11-09T02:22:09,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 6ec486933158f26905c39a8004ccca04 in 891ms, sequenceid=332, compaction requested=true 2024-11-09T02:22:09,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:22:09,214 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:09,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ec486933158f26905c39a8004ccca04:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:22:09,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:09,214 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:09,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ec486933158f26905c39a8004ccca04:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:22:09,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:09,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ec486933158f26905c39a8004ccca04:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:22:09,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:09,215 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:09,215 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 6ec486933158f26905c39a8004ccca04/B is initiating minor compaction (all files) 2024-11-09T02:22:09,215 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:09,215 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): 6ec486933158f26905c39a8004ccca04/A is initiating minor compaction (all files) 2024-11-09T02:22:09,215 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ec486933158f26905c39a8004ccca04/B in TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:09,215 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ec486933158f26905c39a8004ccca04/A in TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:09,216 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/aa88a0ff885e4f11b5f33fa58985c584, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/11d99ad6695c4097b5d142a46e66d539, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/782aa261591449da925f0b20aab61f27] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp, totalSize=36.7 K 2024-11-09T02:22:09,216 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/4d12e269a59e4cbe8542435fd002511c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/1e5de62dc8874b3e8ed39a764bcd3002, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/e18530dad5ba4236b761b196b06a1b7f] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp, totalSize=100.7 K 2024-11-09T02:22:09,216 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:09,216 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. files: [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/4d12e269a59e4cbe8542435fd002511c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/1e5de62dc8874b3e8ed39a764bcd3002, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/e18530dad5ba4236b761b196b06a1b7f] 2024-11-09T02:22:09,216 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting aa88a0ff885e4f11b5f33fa58985c584, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731118926105 2024-11-09T02:22:09,216 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d12e269a59e4cbe8542435fd002511c, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731118926105 2024-11-09T02:22:09,216 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 11d99ad6695c4097b5d142a46e66d539, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731118926811 2024-11-09T02:22:09,217 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e5de62dc8874b3e8ed39a764bcd3002, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731118926811 2024-11-09T02:22:09,217 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting e18530dad5ba4236b761b196b06a1b7f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731118927990 2024-11-09T02:22:09,218 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 782aa261591449da925f0b20aab61f27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731118927994 2024-11-09T02:22:09,226 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ec486933158f26905c39a8004ccca04#B#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:09,227 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/a2691068d7c94d9c92d35829f84b0305 is 50, key is test_row_0/B:col10/1731118927996/Put/seqid=0 2024-11-09T02:22:09,228 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:09,233 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109722feca7f2ab43e6bb8d0e42ef101bd4_6ec486933158f26905c39a8004ccca04 store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:09,235 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109722feca7f2ab43e6bb8d0e42ef101bd4_6ec486933158f26905c39a8004ccca04, store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:09,235 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109722feca7f2ab43e6bb8d0e42ef101bd4_6ec486933158f26905c39a8004ccca04 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:09,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742040_1216 (size=13085) 2024-11-09T02:22:09,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742041_1217 (size=4469) 2024-11-09T02:22:09,248 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ec486933158f26905c39a8004ccca04#A#compaction#178 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:09,249 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/929f7481be7a4a64b2ef4a6bcbad3945 is 175, key is test_row_0/A:col10/1731118927996/Put/seqid=0 2024-11-09T02:22:09,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742042_1218 (size=32039) 2024-11-09T02:22:09,263 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/929f7481be7a4a64b2ef4a6bcbad3945 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/929f7481be7a4a64b2ef4a6bcbad3945 2024-11-09T02:22:09,267 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ec486933158f26905c39a8004ccca04/A of 6ec486933158f26905c39a8004ccca04 into 929f7481be7a4a64b2ef4a6bcbad3945(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:09,267 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:22:09,268 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., storeName=6ec486933158f26905c39a8004ccca04/A, priority=13, startTime=1731118929213; duration=0sec 2024-11-09T02:22:09,268 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:09,268 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ec486933158f26905c39a8004ccca04:A 2024-11-09T02:22:09,268 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:09,269 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:09,269 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): 6ec486933158f26905c39a8004ccca04/C is initiating minor compaction (all files) 2024-11-09T02:22:09,269 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ec486933158f26905c39a8004ccca04/C in TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:09,269 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/0af0f3a784f64337bf20911899e6bbb7, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/f9f53597c0c545968b25fbe6165e626c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/bd41381064d3405499e527dfde900460] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp, totalSize=36.7 K 2024-11-09T02:22:09,269 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0af0f3a784f64337bf20911899e6bbb7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731118926105 2024-11-09T02:22:09,271 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9f53597c0c545968b25fbe6165e626c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731118926811 2024-11-09T02:22:09,272 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd41381064d3405499e527dfde900460, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731118927994 2024-11-09T02:22:09,284 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ec486933158f26905c39a8004ccca04#C#compaction#179 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:09,286 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/cbe297108e674977bb78e8b874fade09 is 50, key is test_row_0/C:col10/1731118927996/Put/seqid=0 2024-11-09T02:22:09,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742043_1219 (size=13085) 2024-11-09T02:22:09,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-09T02:22:09,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:09,330 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 6ec486933158f26905c39a8004ccca04 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-09T02:22:09,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411096d9583cadf76423390d4a904cda10149_6ec486933158f26905c39a8004ccca04 is 50, key is test_row_0/A:col10/1731118928379/Put/seqid=0 2024-11-09T02:22:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T02:22:09,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742044_1220 (size=12454) 2024-11-09T02:22:09,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:22:09,356 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411096d9583cadf76423390d4a904cda10149_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096d9583cadf76423390d4a904cda10149_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:09,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/6629826f9d654aed810bb47f30173fec, store: [table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:09,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/6629826f9d654aed810bb47f30173fec is 175, key is test_row_0/A:col10/1731118928379/Put/seqid=0 2024-11-09T02:22:09,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742045_1221 (size=31255) 2024-11-09T02:22:09,361 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/6629826f9d654aed810bb47f30173fec 2024-11-09T02:22:09,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/3792c7aa64bd46fa90123cd9a4f7b3f2 is 50, key is test_row_0/B:col10/1731118928379/Put/seqid=0 2024-11-09T02:22:09,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742046_1222 (size=12301) 2024-11-09T02:22:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:09,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. as already flushing 2024-11-09T02:22:09,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118989568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:09,645 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/a2691068d7c94d9c92d35829f84b0305 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a2691068d7c94d9c92d35829f84b0305 2024-11-09T02:22:09,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T02:22:09,651 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ec486933158f26905c39a8004ccca04/B of 6ec486933158f26905c39a8004ccca04 into a2691068d7c94d9c92d35829f84b0305(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:09,651 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:22:09,651 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., storeName=6ec486933158f26905c39a8004ccca04/B, priority=13, startTime=1731118929214; duration=0sec 2024-11-09T02:22:09,651 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:09,651 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ec486933158f26905c39a8004ccca04:B 2024-11-09T02:22:09,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:09,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118989679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:09,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:09,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:09,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:09,702 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/cbe297108e674977bb78e8b874fade09 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/cbe297108e674977bb78e8b874fade09 2024-11-09T02:22:09,710 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ec486933158f26905c39a8004ccca04/C of 6ec486933158f26905c39a8004ccca04 into cbe297108e674977bb78e8b874fade09(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:09,710 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:22:09,710 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., storeName=6ec486933158f26905c39a8004ccca04/C, priority=13, startTime=1731118929214; duration=0sec 2024-11-09T02:22:09,710 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:09,710 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ec486933158f26905c39a8004ccca04:C 2024-11-09T02:22:09,775 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/3792c7aa64bd46fa90123cd9a4f7b3f2 2024-11-09T02:22:09,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/cdba1ad0bc164ff8a253090d54844f5c is 50, key is test_row_0/C:col10/1731118928379/Put/seqid=0 2024-11-09T02:22:09,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742047_1223 (size=12301) 2024-11-09T02:22:09,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:09,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118989889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:09,915 INFO [Thread-634 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-634. 2024-11-09T02:22:09,916 DEBUG [Thread-634 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:22:09,916 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:09,916 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:09,916 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:22:09,918 INFO [Thread-632 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-632. 2024-11-09T02:22:09,918 DEBUG [Thread-632 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:22:09,918 DEBUG [Thread-632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:09,918 DEBUG [Thread-632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:09,918 INFO [Thread-638 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-638. 2024-11-09T02:22:09,918 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:22:09,918 DEBUG [Thread-638 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:22:09,918 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:09,918 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:09,918 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:22:09,923 INFO [Thread-636 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-636. 2024-11-09T02:22:09,923 DEBUG [Thread-636 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:22:09,923 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:09,923 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:09,923 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:22:10,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T02:22:10,194 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/cdba1ad0bc164ff8a253090d54844f5c 2024-11-09T02:22:10,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/6629826f9d654aed810bb47f30173fec as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/6629826f9d654aed810bb47f30173fec 2024-11-09T02:22:10,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:10,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48380 deadline: 1731118990200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:10,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:10,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:10,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:10,203 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/6629826f9d654aed810bb47f30173fec, entries=150, sequenceid=353, filesize=30.5 K 2024-11-09T02:22:10,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/3792c7aa64bd46fa90123cd9a4f7b3f2 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/3792c7aa64bd46fa90123cd9a4f7b3f2 2024-11-09T02:22:10,207 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/3792c7aa64bd46fa90123cd9a4f7b3f2, entries=150, sequenceid=353, filesize=12.0 K 2024-11-09T02:22:10,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/cdba1ad0bc164ff8a253090d54844f5c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/cdba1ad0bc164ff8a253090d54844f5c 2024-11-09T02:22:10,212 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/cdba1ad0bc164ff8a253090d54844f5c, entries=150, sequenceid=353, filesize=12.0 K 2024-11-09T02:22:10,213 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 6ec486933158f26905c39a8004ccca04 in 883ms, sequenceid=353, compaction requested=false 2024-11-09T02:22:10,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:22:10,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:10,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-09T02:22:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-09T02:22:10,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-09T02:22:10,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1910 sec 2024-11-09T02:22:10,217 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.1950 sec 2024-11-09T02:22:10,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:10,712 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:22:10,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ec486933158f26905c39a8004ccca04 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-09T02:22:10,712 INFO [Thread-625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-625. 2024-11-09T02:22:10,712 DEBUG [Thread-625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:22:10,712 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:10,712 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:10,712 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:22:10,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109810892f1aa4d4f5cab80676a2992a80f_6ec486933158f26905c39a8004ccca04 is 50, key is test_row_0/A:col10/1731118930710/Put/seqid=0 2024-11-09T02:22:10,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742048_1224 (size=12454) 2024-11-09T02:22:11,123 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:22:11,131 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109810892f1aa4d4f5cab80676a2992a80f_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109810892f1aa4d4f5cab80676a2992a80f_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:11,133 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/6c82a4f4bbe443659a3b421610d88922, store: [table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:11,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/6c82a4f4bbe443659a3b421610d88922 is 175, key is test_row_0/A:col10/1731118930710/Put/seqid=0 2024-11-09T02:22:11,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742049_1225 (size=31255) 2024-11-09T02:22:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T02:22:11,171 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:22:11,541 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=372, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/6c82a4f4bbe443659a3b421610d88922 2024-11-09T02:22:11,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/faeb9f677874424681ab574e3e49fb17 is 50, key is test_row_0/B:col10/1731118930710/Put/seqid=0 2024-11-09T02:22:11,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742050_1226 (size=12301) 2024-11-09T02:22:12,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/faeb9f677874424681ab574e3e49fb17 2024-11-09T02:22:12,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/97c383e914bf46f2b78d417681e9bd77 is 50, key is test_row_0/C:col10/1731118930710/Put/seqid=0 2024-11-09T02:22:12,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742051_1227 (size=12301) 2024-11-09T02:22:12,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/97c383e914bf46f2b78d417681e9bd77 2024-11-09T02:22:12,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/6c82a4f4bbe443659a3b421610d88922 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/6c82a4f4bbe443659a3b421610d88922 2024-11-09T02:22:12,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/6c82a4f4bbe443659a3b421610d88922, entries=150, sequenceid=372, filesize=30.5 K 2024-11-09T02:22:12,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/faeb9f677874424681ab574e3e49fb17 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/faeb9f677874424681ab574e3e49fb17 2024-11-09T02:22:12,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/faeb9f677874424681ab574e3e49fb17, entries=150, sequenceid=372, filesize=12.0 K 2024-11-09T02:22:12,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/97c383e914bf46f2b78d417681e9bd77 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/97c383e914bf46f2b78d417681e9bd77 2024-11-09T02:22:12,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/97c383e914bf46f2b78d417681e9bd77, entries=150, sequenceid=372, filesize=12.0 K 2024-11-09T02:22:12,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=0 B/0 for 6ec486933158f26905c39a8004ccca04 in 1728ms, sequenceid=372, compaction requested=true 2024-11-09T02:22:12,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:22:12,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ec486933158f26905c39a8004ccca04:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:22:12,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:12,440 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:12,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ec486933158f26905c39a8004ccca04:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:22:12,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:12,440 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:12,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ec486933158f26905c39a8004ccca04:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:22:12,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:12,441 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:12,441 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:12,441 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 6ec486933158f26905c39a8004ccca04/B is initiating minor compaction (all files) 2024-11-09T02:22:12,441 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): 6ec486933158f26905c39a8004ccca04/A is initiating minor compaction (all files) 2024-11-09T02:22:12,441 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ec486933158f26905c39a8004ccca04/B in TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:12,441 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ec486933158f26905c39a8004ccca04/A in TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:12,441 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/929f7481be7a4a64b2ef4a6bcbad3945, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/6629826f9d654aed810bb47f30173fec, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/6c82a4f4bbe443659a3b421610d88922] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp, totalSize=92.3 K 2024-11-09T02:22:12,441 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a2691068d7c94d9c92d35829f84b0305, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/3792c7aa64bd46fa90123cd9a4f7b3f2, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/faeb9f677874424681ab574e3e49fb17] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp, totalSize=36.8 K 2024-11-09T02:22:12,441 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:12,442 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. files: [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/929f7481be7a4a64b2ef4a6bcbad3945, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/6629826f9d654aed810bb47f30173fec, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/6c82a4f4bbe443659a3b421610d88922] 2024-11-09T02:22:12,442 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting a2691068d7c94d9c92d35829f84b0305, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731118927994 2024-11-09T02:22:12,442 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 929f7481be7a4a64b2ef4a6bcbad3945, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731118927994 2024-11-09T02:22:12,442 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 3792c7aa64bd46fa90123cd9a4f7b3f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731118928361 2024-11-09T02:22:12,442 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6629826f9d654aed810bb47f30173fec, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731118928361 2024-11-09T02:22:12,442 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting faeb9f677874424681ab574e3e49fb17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731118929558 2024-11-09T02:22:12,442 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c82a4f4bbe443659a3b421610d88922, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731118929558 2024-11-09T02:22:12,452 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:12,454 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ec486933158f26905c39a8004ccca04#B#compaction#186 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:12,454 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/baf27cd9a0b841e393651b57e32a2818 is 50, key is test_row_0/B:col10/1731118930710/Put/seqid=0 2024-11-09T02:22:12,455 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110943479ea7266e405e927a65ac765b7d4f_6ec486933158f26905c39a8004ccca04 store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:12,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742052_1228 (size=13187) 2024-11-09T02:22:12,489 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110943479ea7266e405e927a65ac765b7d4f_6ec486933158f26905c39a8004ccca04, store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:12,489 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110943479ea7266e405e927a65ac765b7d4f_6ec486933158f26905c39a8004ccca04 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:12,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742053_1229 (size=4469) 2024-11-09T02:22:12,673 INFO [Thread-629 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-629. 2024-11-09T02:22:12,673 DEBUG [Thread-629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:22:12,673 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:12,673 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:12,673 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:22:12,685 INFO [Thread-621 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-621. 2024-11-09T02:22:12,685 DEBUG [Thread-621 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:22:12,685 DEBUG [Thread-621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:12,685 DEBUG [Thread-621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:12,685 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:22:12,868 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/baf27cd9a0b841e393651b57e32a2818 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/baf27cd9a0b841e393651b57e32a2818 2024-11-09T02:22:12,873 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ec486933158f26905c39a8004ccca04/B of 6ec486933158f26905c39a8004ccca04 into baf27cd9a0b841e393651b57e32a2818(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:12,873 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:22:12,873 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., storeName=6ec486933158f26905c39a8004ccca04/B, priority=13, startTime=1731118932440; duration=0sec 2024-11-09T02:22:12,873 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:12,873 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ec486933158f26905c39a8004ccca04:B 2024-11-09T02:22:12,873 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:12,874 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:12,874 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 6ec486933158f26905c39a8004ccca04/C is initiating minor compaction (all files) 2024-11-09T02:22:12,874 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ec486933158f26905c39a8004ccca04/C in TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:12,875 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/cbe297108e674977bb78e8b874fade09, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/cdba1ad0bc164ff8a253090d54844f5c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/97c383e914bf46f2b78d417681e9bd77] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp, totalSize=36.8 K 2024-11-09T02:22:12,875 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting cbe297108e674977bb78e8b874fade09, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731118927994 2024-11-09T02:22:12,875 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting cdba1ad0bc164ff8a253090d54844f5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731118928361 2024-11-09T02:22:12,876 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 97c383e914bf46f2b78d417681e9bd77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731118929558 2024-11-09T02:22:12,884 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ec486933158f26905c39a8004ccca04#C#compaction#188 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:12,884 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/05dc2b5a122b4245a14f447b77648aac is 50, key is test_row_0/C:col10/1731118930710/Put/seqid=0 2024-11-09T02:22:12,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742054_1230 (size=13187) 2024-11-09T02:22:12,895 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ec486933158f26905c39a8004ccca04#A#compaction#187 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:12,895 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/9f92b7d6a00e4b5d8befe17829bed44c is 175, key is test_row_0/A:col10/1731118930710/Put/seqid=0 2024-11-09T02:22:12,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742055_1231 (size=32141) 2024-11-09T02:22:13,293 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/05dc2b5a122b4245a14f447b77648aac as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/05dc2b5a122b4245a14f447b77648aac 2024-11-09T02:22:13,298 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ec486933158f26905c39a8004ccca04/C of 6ec486933158f26905c39a8004ccca04 into 05dc2b5a122b4245a14f447b77648aac(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:13,298 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:22:13,298 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., storeName=6ec486933158f26905c39a8004ccca04/C, priority=13, startTime=1731118932440; duration=0sec 2024-11-09T02:22:13,298 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:13,298 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ec486933158f26905c39a8004ccca04:C 2024-11-09T02:22:13,304 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/9f92b7d6a00e4b5d8befe17829bed44c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/9f92b7d6a00e4b5d8befe17829bed44c 2024-11-09T02:22:13,309 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ec486933158f26905c39a8004ccca04/A of 6ec486933158f26905c39a8004ccca04 into 9f92b7d6a00e4b5d8befe17829bed44c(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:13,309 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ec486933158f26905c39a8004ccca04: 2024-11-09T02:22:13,309 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., storeName=6ec486933158f26905c39a8004ccca04/A, priority=13, startTime=1731118932440; duration=0sec 2024-11-09T02:22:13,309 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:13,309 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ec486933158f26905c39a8004ccca04:A 2024-11-09T02:22:13,498 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T02:22:17,584 INFO [Thread-623 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-623. 2024-11-09T02:22:17,584 DEBUG [Thread-623 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:22:17,584 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:17,585 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:17,585 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:22:17,603 INFO [Thread-627 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-627. 2024-11-09T02:22:17,603 DEBUG [Thread-627 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:22:17,603 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:17,603 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:17,603 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:22:17,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T02:22:17,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-09T02:22:17,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-09T02:22:17,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-09T02:22:17,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-09T02:22:17,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-09T02:22:17,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T02:22:17,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3923 2024-11-09T02:22:17,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4051 2024-11-09T02:22:17,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T02:22:17,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1733 2024-11-09T02:22:17,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5196 rows 2024-11-09T02:22:17,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1738 2024-11-09T02:22:17,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5210 rows 2024-11-09T02:22:17,603 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T02:22:17,603 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:22:17,603 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:17,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:17,604 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:22:17,604 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T02:22:17,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T02:22:17,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-09T02:22:17,607 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118937607"}]},"ts":"1731118937607"} 2024-11-09T02:22:17,609 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T02:22:17,609 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T02:22:17,610 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T02:22:17,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ec486933158f26905c39a8004ccca04, UNASSIGN}] 2024-11-09T02:22:17,612 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ec486933158f26905c39a8004ccca04, UNASSIGN 2024-11-09T02:22:17,612 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=6ec486933158f26905c39a8004ccca04, regionState=CLOSING, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:17,614 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ec486933158f26905c39a8004ccca04, UNASSIGN because future has completed 2024-11-09T02:22:17,614 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T02:22:17,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223}] 2024-11-09T02:22:17,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-09T02:22:17,766 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close 6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:17,767 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T02:22:17,767 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing 6ec486933158f26905c39a8004ccca04, disabling compactions & flushes 2024-11-09T02:22:17,767 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:17,767 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:17,767 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. after waiting 0 ms 2024-11-09T02:22:17,767 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:17,767 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing 6ec486933158f26905c39a8004ccca04 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-09T02:22:17,773 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109743450f739e94e6eaea5a127ce65fea8_6ec486933158f26905c39a8004ccca04 is 50, key is test_row_0/A:col10/1731118937580/Put/seqid=0 2024-11-09T02:22:17,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742056_1232 (size=9914) 2024-11-09T02:22:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-09T02:22:18,196 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:22:18,201 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109743450f739e94e6eaea5a127ce65fea8_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109743450f739e94e6eaea5a127ce65fea8_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:18,202 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/25299a84ede34ad9a2585be8a978c3cb, store: [table=TestAcidGuarantees family=A region=6ec486933158f26905c39a8004ccca04] 2024-11-09T02:22:18,203 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/25299a84ede34ad9a2585be8a978c3cb is 175, key is test_row_0/A:col10/1731118937580/Put/seqid=0 2024-11-09T02:22:18,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742057_1233 (size=22561) 2024-11-09T02:22:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-09T02:22:18,607 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=382, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/25299a84ede34ad9a2585be8a978c3cb 2024-11-09T02:22:18,613 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/ff691503342b42ea81f9b52995c607b6 is 50, key is test_row_0/B:col10/1731118937580/Put/seqid=0 2024-11-09T02:22:18,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742058_1234 (size=9857) 2024-11-09T02:22:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-09T02:22:19,017 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/ff691503342b42ea81f9b52995c607b6 2024-11-09T02:22:19,024 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/bf9a55199e1747c7bd355b54420f3f66 is 50, key is test_row_0/C:col10/1731118937580/Put/seqid=0 2024-11-09T02:22:19,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742059_1235 (size=9857) 2024-11-09T02:22:19,428 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/bf9a55199e1747c7bd355b54420f3f66 2024-11-09T02:22:19,433 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/A/25299a84ede34ad9a2585be8a978c3cb as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/25299a84ede34ad9a2585be8a978c3cb 2024-11-09T02:22:19,437 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/25299a84ede34ad9a2585be8a978c3cb, entries=100, sequenceid=382, filesize=22.0 K 2024-11-09T02:22:19,438 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/B/ff691503342b42ea81f9b52995c607b6 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/ff691503342b42ea81f9b52995c607b6 2024-11-09T02:22:19,442 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/ff691503342b42ea81f9b52995c607b6, entries=100, sequenceid=382, filesize=9.6 K 2024-11-09T02:22:19,443 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/.tmp/C/bf9a55199e1747c7bd355b54420f3f66 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/bf9a55199e1747c7bd355b54420f3f66 2024-11-09T02:22:19,447 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/bf9a55199e1747c7bd355b54420f3f66, entries=100, sequenceid=382, filesize=9.6 K 2024-11-09T02:22:19,448 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 6ec486933158f26905c39a8004ccca04 in 1681ms, sequenceid=382, compaction requested=false 2024-11-09T02:22:19,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/79ccbf69fd5c42f9acd2d9546cb5168c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/8c492854e0b74361b0ea1f9daaaa026e, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/3417402b38db48fea85429e450420471, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/d02e875e8e96433eb0baa40027ee8c6b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/d3d9c81e80e94d3b94b5754abdbe6b72, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/de4c7e99259146519557c6a21cfc99b3, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/3b146ba2d2fa47f1ac0d9f2f8fdc0ed5, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/f76b4d8e272c4b0d810f07b9214c5c82, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/ec1a6c58c983480287b59208d131ade2, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/b48dd14ba87b40ceaccb2e025d0517e4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/3a1eeecd393449ff97b8f211c8aa09cc, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/022c60ae221b47959fd2fd6876abb34c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/5547e456856542ee8d5ed9e805f589a9, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/e97de2f34bed44638a5adf888fa73aa4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/0ed38f6b96f6450995489883ef2cd3ff, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/042ae56d26f24141b73d8ecca6d3c1de, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/4593cb74158e4801948478d15e1c91d0, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/7f367d756b474e31aa07e47ee294f60b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/a70320c2ed784b8f9fd605782831aab3, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/f0d475cae768481fabc26f74badd6926, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/4d12e269a59e4cbe8542435fd002511c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/1e5de62dc8874b3e8ed39a764bcd3002, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/e18530dad5ba4236b761b196b06a1b7f, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/929f7481be7a4a64b2ef4a6bcbad3945, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/6629826f9d654aed810bb47f30173fec, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/6c82a4f4bbe443659a3b421610d88922] to archive 2024-11-09T02:22:19,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T02:22:19,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/79ccbf69fd5c42f9acd2d9546cb5168c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/79ccbf69fd5c42f9acd2d9546cb5168c 2024-11-09T02:22:19,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/8c492854e0b74361b0ea1f9daaaa026e to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/8c492854e0b74361b0ea1f9daaaa026e 2024-11-09T02:22:19,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/3417402b38db48fea85429e450420471 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/3417402b38db48fea85429e450420471 2024-11-09T02:22:19,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/d02e875e8e96433eb0baa40027ee8c6b to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/d02e875e8e96433eb0baa40027ee8c6b 2024-11-09T02:22:19,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/d3d9c81e80e94d3b94b5754abdbe6b72 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/d3d9c81e80e94d3b94b5754abdbe6b72 2024-11-09T02:22:19,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/de4c7e99259146519557c6a21cfc99b3 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/de4c7e99259146519557c6a21cfc99b3 2024-11-09T02:22:19,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/3b146ba2d2fa47f1ac0d9f2f8fdc0ed5 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/3b146ba2d2fa47f1ac0d9f2f8fdc0ed5 2024-11-09T02:22:19,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/f76b4d8e272c4b0d810f07b9214c5c82 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/f76b4d8e272c4b0d810f07b9214c5c82 2024-11-09T02:22:19,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/ec1a6c58c983480287b59208d131ade2 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/ec1a6c58c983480287b59208d131ade2 2024-11-09T02:22:19,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/b48dd14ba87b40ceaccb2e025d0517e4 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/b48dd14ba87b40ceaccb2e025d0517e4 2024-11-09T02:22:19,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/3a1eeecd393449ff97b8f211c8aa09cc to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/3a1eeecd393449ff97b8f211c8aa09cc 2024-11-09T02:22:19,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/022c60ae221b47959fd2fd6876abb34c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/022c60ae221b47959fd2fd6876abb34c 2024-11-09T02:22:19,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/5547e456856542ee8d5ed9e805f589a9 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/5547e456856542ee8d5ed9e805f589a9 2024-11-09T02:22:19,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/e97de2f34bed44638a5adf888fa73aa4 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/e97de2f34bed44638a5adf888fa73aa4 2024-11-09T02:22:19,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/0ed38f6b96f6450995489883ef2cd3ff to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/0ed38f6b96f6450995489883ef2cd3ff 2024-11-09T02:22:19,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/042ae56d26f24141b73d8ecca6d3c1de to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/042ae56d26f24141b73d8ecca6d3c1de 2024-11-09T02:22:19,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/4593cb74158e4801948478d15e1c91d0 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/4593cb74158e4801948478d15e1c91d0 2024-11-09T02:22:19,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/7f367d756b474e31aa07e47ee294f60b to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/7f367d756b474e31aa07e47ee294f60b 2024-11-09T02:22:19,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/a70320c2ed784b8f9fd605782831aab3 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/a70320c2ed784b8f9fd605782831aab3 2024-11-09T02:22:19,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/f0d475cae768481fabc26f74badd6926 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/f0d475cae768481fabc26f74badd6926 2024-11-09T02:22:19,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/4d12e269a59e4cbe8542435fd002511c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/4d12e269a59e4cbe8542435fd002511c 2024-11-09T02:22:19,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/1e5de62dc8874b3e8ed39a764bcd3002 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/1e5de62dc8874b3e8ed39a764bcd3002 2024-11-09T02:22:19,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/e18530dad5ba4236b761b196b06a1b7f to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/e18530dad5ba4236b761b196b06a1b7f 2024-11-09T02:22:19,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/929f7481be7a4a64b2ef4a6bcbad3945 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/929f7481be7a4a64b2ef4a6bcbad3945 2024-11-09T02:22:19,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/6629826f9d654aed810bb47f30173fec to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/6629826f9d654aed810bb47f30173fec 2024-11-09T02:22:19,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/6c82a4f4bbe443659a3b421610d88922 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/6c82a4f4bbe443659a3b421610d88922 2024-11-09T02:22:19,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/2f3d04ad3cd3438ebb75149004e5dad3, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/aad047c6aef54698aa5e93e4d363f3eb, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/19ca8b3d2ca6438b93e9eac575dba010, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/bc393ce77e554fcda79fa8873c8bcb19, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/341615a7240c47d39e160b5a600c4986, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a4908276de084821bc2dd66130139244, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/bf1d4376bf8846258f30e5c0bc9843fa, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/40b05c6c646546778922da72237185e9, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/1940603241854816ba35a1c6ad78c23d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/3b74eb81937e45efaceb910607100fd5, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a4b96985338244dd95d0a22e2581949f, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/f98df9a7e48748ae91b98c780f45ec8e, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/425371313f2f42f3bf693b9b5599c62e, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/589e0ed6cbe54f70b3aea3eb9cedb69b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/166bfb09b8ce48bc8ea17f6c971383f8, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/6d9fe293b9b64ff7a6354bca541016c4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a1afb25d9af44f7a97d1c5591fc1f780, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/17172f5939a24f18bba6f022fe0a42c5, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/101dca799d5b47078d3a1680e30dbbd1, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/aa88a0ff885e4f11b5f33fa58985c584, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a692e5ba057946f1a71f0d2770793672, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/11d99ad6695c4097b5d142a46e66d539, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a2691068d7c94d9c92d35829f84b0305, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/782aa261591449da925f0b20aab61f27, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/3792c7aa64bd46fa90123cd9a4f7b3f2, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/faeb9f677874424681ab574e3e49fb17] to archive 2024-11-09T02:22:19,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T02:22:19,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/2f3d04ad3cd3438ebb75149004e5dad3 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/2f3d04ad3cd3438ebb75149004e5dad3 2024-11-09T02:22:19,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/aad047c6aef54698aa5e93e4d363f3eb to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/aad047c6aef54698aa5e93e4d363f3eb 2024-11-09T02:22:19,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/19ca8b3d2ca6438b93e9eac575dba010 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/19ca8b3d2ca6438b93e9eac575dba010 2024-11-09T02:22:19,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/bc393ce77e554fcda79fa8873c8bcb19 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/bc393ce77e554fcda79fa8873c8bcb19 2024-11-09T02:22:19,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/341615a7240c47d39e160b5a600c4986 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/341615a7240c47d39e160b5a600c4986 2024-11-09T02:22:19,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a4908276de084821bc2dd66130139244 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a4908276de084821bc2dd66130139244 2024-11-09T02:22:19,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/bf1d4376bf8846258f30e5c0bc9843fa to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/bf1d4376bf8846258f30e5c0bc9843fa 2024-11-09T02:22:19,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/40b05c6c646546778922da72237185e9 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/40b05c6c646546778922da72237185e9 2024-11-09T02:22:19,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/1940603241854816ba35a1c6ad78c23d to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/1940603241854816ba35a1c6ad78c23d 2024-11-09T02:22:19,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/3b74eb81937e45efaceb910607100fd5 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/3b74eb81937e45efaceb910607100fd5 2024-11-09T02:22:19,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a4b96985338244dd95d0a22e2581949f to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a4b96985338244dd95d0a22e2581949f 2024-11-09T02:22:19,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/f98df9a7e48748ae91b98c780f45ec8e to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/f98df9a7e48748ae91b98c780f45ec8e 2024-11-09T02:22:19,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/425371313f2f42f3bf693b9b5599c62e to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/425371313f2f42f3bf693b9b5599c62e 2024-11-09T02:22:19,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/589e0ed6cbe54f70b3aea3eb9cedb69b to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/589e0ed6cbe54f70b3aea3eb9cedb69b 2024-11-09T02:22:19,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/166bfb09b8ce48bc8ea17f6c971383f8 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/166bfb09b8ce48bc8ea17f6c971383f8 2024-11-09T02:22:19,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/6d9fe293b9b64ff7a6354bca541016c4 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/6d9fe293b9b64ff7a6354bca541016c4 2024-11-09T02:22:19,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a1afb25d9af44f7a97d1c5591fc1f780 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a1afb25d9af44f7a97d1c5591fc1f780 2024-11-09T02:22:19,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/17172f5939a24f18bba6f022fe0a42c5 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/17172f5939a24f18bba6f022fe0a42c5 2024-11-09T02:22:19,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/101dca799d5b47078d3a1680e30dbbd1 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/101dca799d5b47078d3a1680e30dbbd1 2024-11-09T02:22:19,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/aa88a0ff885e4f11b5f33fa58985c584 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/aa88a0ff885e4f11b5f33fa58985c584 2024-11-09T02:22:19,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a692e5ba057946f1a71f0d2770793672 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a692e5ba057946f1a71f0d2770793672 2024-11-09T02:22:19,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/11d99ad6695c4097b5d142a46e66d539 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/11d99ad6695c4097b5d142a46e66d539 2024-11-09T02:22:19,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a2691068d7c94d9c92d35829f84b0305 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/a2691068d7c94d9c92d35829f84b0305 2024-11-09T02:22:19,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/782aa261591449da925f0b20aab61f27 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/782aa261591449da925f0b20aab61f27 2024-11-09T02:22:19,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/3792c7aa64bd46fa90123cd9a4f7b3f2 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/3792c7aa64bd46fa90123cd9a4f7b3f2 2024-11-09T02:22:19,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/faeb9f677874424681ab574e3e49fb17 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/faeb9f677874424681ab574e3e49fb17 2024-11-09T02:22:19,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/0fdb1985042e412d955872cc8fc8017d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/5cda97cc16074df1b4bb7a8d64d60135, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/7da40e7ddea54667b36e4c84c64dd5a5, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/268264bdc7634397a187f60f5f62cb05, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/6c96e7e9e5cd4e1eb115d2f2231aec36, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/391676a9990e4c7581d87608c16d1848, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/1871b50daa3c4796b78b3b3168e14984, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/e01908720f80419096678d4da7c428df, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/ebc9dff3a34640fe84dd071736636b92, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/c95334adfaad4942a82a98747f4dcc22, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/62db74bccb4445209ec5b8f9271ef1af, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/4f0d323f80674378a0bc841cfcf2e50a, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/1fed847ea39246e6908887f0a182e8d2, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/284e02e9227c434b881724c3dc8b4135, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/d081089af75b48228276268eb7685c83, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/662e1ff85b7942afb7c2a7ee3f59d777, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/2d45d55253cf4a52926eb08d281d0f8d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/8f9eef52699e490c891e1ec73380253d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/60ef84ccc5eb4018b238e52d406ea774, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/0af0f3a784f64337bf20911899e6bbb7, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/8d2346897e4b4b0685633fe373c8cef1, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/f9f53597c0c545968b25fbe6165e626c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/cbe297108e674977bb78e8b874fade09, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/bd41381064d3405499e527dfde900460, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/cdba1ad0bc164ff8a253090d54844f5c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/97c383e914bf46f2b78d417681e9bd77] to archive 2024-11-09T02:22:19,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T02:22:19,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/0fdb1985042e412d955872cc8fc8017d to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/0fdb1985042e412d955872cc8fc8017d 2024-11-09T02:22:19,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/5cda97cc16074df1b4bb7a8d64d60135 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/5cda97cc16074df1b4bb7a8d64d60135 2024-11-09T02:22:19,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/7da40e7ddea54667b36e4c84c64dd5a5 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/7da40e7ddea54667b36e4c84c64dd5a5 2024-11-09T02:22:19,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/268264bdc7634397a187f60f5f62cb05 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/268264bdc7634397a187f60f5f62cb05 2024-11-09T02:22:19,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/6c96e7e9e5cd4e1eb115d2f2231aec36 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/6c96e7e9e5cd4e1eb115d2f2231aec36 2024-11-09T02:22:19,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/391676a9990e4c7581d87608c16d1848 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/391676a9990e4c7581d87608c16d1848 2024-11-09T02:22:19,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/1871b50daa3c4796b78b3b3168e14984 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/1871b50daa3c4796b78b3b3168e14984 2024-11-09T02:22:19,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/e01908720f80419096678d4da7c428df to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/e01908720f80419096678d4da7c428df 2024-11-09T02:22:19,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/ebc9dff3a34640fe84dd071736636b92 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/ebc9dff3a34640fe84dd071736636b92 2024-11-09T02:22:19,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/c95334adfaad4942a82a98747f4dcc22 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/c95334adfaad4942a82a98747f4dcc22 2024-11-09T02:22:19,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/62db74bccb4445209ec5b8f9271ef1af to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/62db74bccb4445209ec5b8f9271ef1af 2024-11-09T02:22:19,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/4f0d323f80674378a0bc841cfcf2e50a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/4f0d323f80674378a0bc841cfcf2e50a 2024-11-09T02:22:19,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/1fed847ea39246e6908887f0a182e8d2 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/1fed847ea39246e6908887f0a182e8d2 2024-11-09T02:22:19,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/284e02e9227c434b881724c3dc8b4135 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/284e02e9227c434b881724c3dc8b4135 2024-11-09T02:22:19,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/d081089af75b48228276268eb7685c83 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/d081089af75b48228276268eb7685c83 2024-11-09T02:22:19,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/662e1ff85b7942afb7c2a7ee3f59d777 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/662e1ff85b7942afb7c2a7ee3f59d777 2024-11-09T02:22:19,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/2d45d55253cf4a52926eb08d281d0f8d to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/2d45d55253cf4a52926eb08d281d0f8d 2024-11-09T02:22:19,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/8f9eef52699e490c891e1ec73380253d to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/8f9eef52699e490c891e1ec73380253d 2024-11-09T02:22:19,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/60ef84ccc5eb4018b238e52d406ea774 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/60ef84ccc5eb4018b238e52d406ea774 2024-11-09T02:22:19,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/0af0f3a784f64337bf20911899e6bbb7 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/0af0f3a784f64337bf20911899e6bbb7 2024-11-09T02:22:19,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/8d2346897e4b4b0685633fe373c8cef1 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/8d2346897e4b4b0685633fe373c8cef1 2024-11-09T02:22:19,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/f9f53597c0c545968b25fbe6165e626c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/f9f53597c0c545968b25fbe6165e626c 2024-11-09T02:22:19,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/cbe297108e674977bb78e8b874fade09 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/cbe297108e674977bb78e8b874fade09 2024-11-09T02:22:19,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/bd41381064d3405499e527dfde900460 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/bd41381064d3405499e527dfde900460 2024-11-09T02:22:19,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/cdba1ad0bc164ff8a253090d54844f5c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/cdba1ad0bc164ff8a253090d54844f5c 2024-11-09T02:22:19,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/97c383e914bf46f2b78d417681e9bd77 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/97c383e914bf46f2b78d417681e9bd77 2024-11-09T02:22:19,546 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/recovered.edits/385.seqid, newMaxSeqId=385, maxSeqId=4 2024-11-09T02:22:19,547 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. 2024-11-09T02:22:19,547 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for 6ec486933158f26905c39a8004ccca04: Waiting for close lock at 1731118937767Running coprocessor pre-close hooks at 1731118937767Disabling compacts and flushes for region at 1731118937767Disabling writes for close at 1731118937767Obtaining lock to block concurrent updates at 1731118937767Preparing flush snapshotting stores in 6ec486933158f26905c39a8004ccca04 at 1731118937767Finished memstore snapshotting TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731118937767Flushing stores of TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04. at 1731118937768 (+1 ms)Flushing 6ec486933158f26905c39a8004ccca04/A: creating writer at 1731118937768Flushing mob file 6ec486933158f26905c39a8004ccca04/A: appending metadata at 1731118937773 (+5 ms)Flushing mob file 6ec486933158f26905c39a8004ccca04/A: closing flushed file at 1731118937773Flushing 6ec486933158f26905c39a8004ccca04/A: appending metadata at 1731118938202 (+429 ms)Flushing 6ec486933158f26905c39a8004ccca04/A: closing flushed file at 1731118938202Flushing 6ec486933158f26905c39a8004ccca04/B: creating writer at 1731118938610 (+408 ms)Flushing 6ec486933158f26905c39a8004ccca04/B: appending metadata at 1731118938613 (+3 ms)Flushing 6ec486933158f26905c39a8004ccca04/B: closing flushed file at 1731118938613Flushing 6ec486933158f26905c39a8004ccca04/C: creating writer at 1731118939021 (+408 ms)Flushing 6ec486933158f26905c39a8004ccca04/C: appending metadata at 1731118939024 (+3 ms)Flushing 6ec486933158f26905c39a8004ccca04/C: closing flushed file at 1731118939024Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22870592: reopening flushed file at 1731118939432 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73d68a7: reopening flushed file at 1731118939437 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60bfd391: reopening flushed file at 1731118939442 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 6ec486933158f26905c39a8004ccca04 in 1681ms, sequenceid=382, compaction requested=false at 1731118939448 (+6 ms)Writing region close event to WAL at 1731118939543 (+95 ms)Running coprocessor post-close hooks at 1731118939547 (+4 ms)Closed at 1731118939547 2024-11-09T02:22:19,549 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed 6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:19,549 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=6ec486933158f26905c39a8004ccca04, regionState=CLOSED 2024-11-09T02:22:19,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 because future has completed 2024-11-09T02:22:19,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-09T02:22:19,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ec486933158f26905c39a8004ccca04, server=53f286d3af3b,46079,1731118875223 in 1.9370 sec 2024-11-09T02:22:19,554 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-09T02:22:19,555 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ec486933158f26905c39a8004ccca04, UNASSIGN in 1.9420 sec 2024-11-09T02:22:19,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-09T02:22:19,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9450 sec 2024-11-09T02:22:19,558 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118939558"}]},"ts":"1731118939558"} 2024-11-09T02:22:19,559 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T02:22:19,559 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T02:22:19,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9560 sec 2024-11-09T02:22:19,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-09T02:22:19,750 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:22:19,751 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T02:22:19,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:22:19,752 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:22:19,753 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:22:19,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-09T02:22:19,755 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:19,758 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A, FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B, FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C, FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/recovered.edits] 2024-11-09T02:22:19,760 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/25299a84ede34ad9a2585be8a978c3cb to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/25299a84ede34ad9a2585be8a978c3cb 2024-11-09T02:22:19,762 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/9f92b7d6a00e4b5d8befe17829bed44c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/A/9f92b7d6a00e4b5d8befe17829bed44c 2024-11-09T02:22:19,764 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/baf27cd9a0b841e393651b57e32a2818 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/baf27cd9a0b841e393651b57e32a2818 2024-11-09T02:22:19,765 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/ff691503342b42ea81f9b52995c607b6 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/B/ff691503342b42ea81f9b52995c607b6 2024-11-09T02:22:19,767 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/05dc2b5a122b4245a14f447b77648aac to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/05dc2b5a122b4245a14f447b77648aac 2024-11-09T02:22:19,768 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/bf9a55199e1747c7bd355b54420f3f66 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/C/bf9a55199e1747c7bd355b54420f3f66 2024-11-09T02:22:19,771 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/recovered.edits/385.seqid to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04/recovered.edits/385.seqid 2024-11-09T02:22:19,772 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:19,772 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T02:22:19,772 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T02:22:19,773 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-09T02:22:19,777 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090599c2310caf4b198f391bdc9ca501f9_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090599c2310caf4b198f391bdc9ca501f9_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:19,778 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109118af24d0bb64c61ad6885cb00c1cc41_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109118af24d0bb64c61ad6885cb00c1cc41_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:19,779 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091806af680f8f41ee836e81db56b314f7_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091806af680f8f41ee836e81db56b314f7_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:19,781 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092f048f1a4d244d81bac7e5176a68ce87_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092f048f1a4d244d81bac7e5176a68ce87_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:19,782 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110938b6db5120be448c9941a2571df61341_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110938b6db5120be448c9941a2571df61341_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:19,783 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110949139ba58f394088b700254b9e6b70f7_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110949139ba58f394088b700254b9e6b70f7_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:19,784 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096d9583cadf76423390d4a904cda10149_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096d9583cadf76423390d4a904cda10149_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:19,785 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109743450f739e94e6eaea5a127ce65fea8_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109743450f739e94e6eaea5a127ce65fea8_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:19,786 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109810892f1aa4d4f5cab80676a2992a80f_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109810892f1aa4d4f5cab80676a2992a80f_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:19,787 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110985371048506841d492b091e8e06821ce_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110985371048506841d492b091e8e06821ce_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:19,788 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110988e574bd200f401987680ca346d779ed_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110988e574bd200f401987680ca346d779ed_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:19,790 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411098a9fa6db1c9a42729c00c0173fbd1c78_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411098a9fa6db1c9a42729c00c0173fbd1c78_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:19,791 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110996ffc92daac24c2892678080c2f14fac_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110996ffc92daac24c2892678080c2f14fac_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:19,792 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109aed33e4f790547648fbffb1f22b3eff4_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109aed33e4f790547648fbffb1f22b3eff4_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:19,793 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c21058c1f6964cecb03b36e07256ecd3_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c21058c1f6964cecb03b36e07256ecd3_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:19,794 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ccd2c8fbd32c4ac4a27459a0a4356895_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ccd2c8fbd32c4ac4a27459a0a4356895_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:19,795 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f0bb6a2623054debb5a19c7cf371629c_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f0bb6a2623054debb5a19c7cf371629c_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:19,796 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f3fc8e7fc9584cd7b45a295c55c8926d_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f3fc8e7fc9584cd7b45a295c55c8926d_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:19,797 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f6da6745897641d9a4ea412e0f5c500d_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f6da6745897641d9a4ea412e0f5c500d_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:19,798 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109fb58ba38dc374ea796991e7a84d0164e_6ec486933158f26905c39a8004ccca04 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109fb58ba38dc374ea796991e7a84d0164e_6ec486933158f26905c39a8004ccca04 2024-11-09T02:22:19,799 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T02:22:19,801 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:22:19,803 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T02:22:19,805 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T02:22:19,806 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:22:19,807 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T02:22:19,807 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731118939807"}]},"ts":"9223372036854775807"} 2024-11-09T02:22:19,809 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T02:22:19,809 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6ec486933158f26905c39a8004ccca04, NAME => 'TestAcidGuarantees,,1731118908252.6ec486933158f26905c39a8004ccca04.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T02:22:19,809 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T02:22:19,809 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731118939809"}]},"ts":"9223372036854775807"} 2024-11-09T02:22:19,811 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T02:22:19,811 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:22:19,812 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 60 msec 2024-11-09T02:22:19,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-09T02:22:19,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T02:22:19,860 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:22:19,869 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 237) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1192768363_22 at /127.0.0.1:50174 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/cluster_76eb7638-e5d7-2221-3a01-3d32335e942f/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/cluster_76eb7638-e5d7-2221-3a01-3d32335e942f/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=525 (was 613), ProcessCount=11 (was 11), AvailableMemoryMB=2885 (was 3182) 2024-11-09T02:22:19,879 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=525, ProcessCount=11, AvailableMemoryMB=2884 2024-11-09T02:22:19,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T02:22:19,880 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:22:19,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T02:22:19,883 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:22:19,883 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:22:19,883 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-09T02:22:19,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-09T02:22:19,884 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:22:19,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742060_1236 (size=959) 2024-11-09T02:22:19,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-09T02:22:20,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-09T02:22:20,293 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a 2024-11-09T02:22:20,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742061_1237 (size=53) 2024-11-09T02:22:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-09T02:22:20,709 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:22:20,709 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fbbc71fa787b8911356ef7975af46e80, disabling compactions & flushes 2024-11-09T02:22:20,709 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:20,709 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:20,709 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. after waiting 0 ms 2024-11-09T02:22:20,709 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:20,709 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:20,709 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fbbc71fa787b8911356ef7975af46e80: Waiting for close lock at 1731118940709Disabling compacts and flushes for region at 1731118940709Disabling writes for close at 1731118940709Writing region close event to WAL at 1731118940709Closed at 1731118940709 2024-11-09T02:22:20,711 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:22:20,712 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731118940711"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118940711"}]},"ts":"1731118940711"} 2024-11-09T02:22:20,715 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T02:22:20,717 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:22:20,718 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118940717"}]},"ts":"1731118940717"} 2024-11-09T02:22:20,720 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T02:22:20,720 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fbbc71fa787b8911356ef7975af46e80, ASSIGN}] 2024-11-09T02:22:20,721 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fbbc71fa787b8911356ef7975af46e80, ASSIGN 2024-11-09T02:22:20,722 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fbbc71fa787b8911356ef7975af46e80, ASSIGN; state=OFFLINE, location=53f286d3af3b,46079,1731118875223; forceNewPlan=false, retain=false 2024-11-09T02:22:20,873 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=fbbc71fa787b8911356ef7975af46e80, regionState=OPENING, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:20,876 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fbbc71fa787b8911356ef7975af46e80, ASSIGN because future has completed 2024-11-09T02:22:20,876 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223}] 2024-11-09T02:22:21,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-09T02:22:21,031 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:21,031 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:22:21,032 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:21,032 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:22:21,032 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:21,032 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:21,034 INFO [StoreOpener-fbbc71fa787b8911356ef7975af46e80-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:21,036 INFO [StoreOpener-fbbc71fa787b8911356ef7975af46e80-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbbc71fa787b8911356ef7975af46e80 columnFamilyName A 2024-11-09T02:22:21,036 DEBUG [StoreOpener-fbbc71fa787b8911356ef7975af46e80-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:22:21,037 INFO [StoreOpener-fbbc71fa787b8911356ef7975af46e80-1 {}] regionserver.HStore(327): Store=fbbc71fa787b8911356ef7975af46e80/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:22:21,037 INFO [StoreOpener-fbbc71fa787b8911356ef7975af46e80-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:21,039 INFO [StoreOpener-fbbc71fa787b8911356ef7975af46e80-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbbc71fa787b8911356ef7975af46e80 columnFamilyName B 2024-11-09T02:22:21,039 DEBUG [StoreOpener-fbbc71fa787b8911356ef7975af46e80-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:22:21,040 INFO [StoreOpener-fbbc71fa787b8911356ef7975af46e80-1 {}] regionserver.HStore(327): Store=fbbc71fa787b8911356ef7975af46e80/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:22:21,040 INFO [StoreOpener-fbbc71fa787b8911356ef7975af46e80-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:21,042 INFO [StoreOpener-fbbc71fa787b8911356ef7975af46e80-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbbc71fa787b8911356ef7975af46e80 columnFamilyName C 2024-11-09T02:22:21,042 DEBUG [StoreOpener-fbbc71fa787b8911356ef7975af46e80-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:22:21,042 INFO [StoreOpener-fbbc71fa787b8911356ef7975af46e80-1 {}] regionserver.HStore(327): Store=fbbc71fa787b8911356ef7975af46e80/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:22:21,043 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:21,043 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:21,044 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:21,044 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:21,044 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:21,045 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T02:22:21,046 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:21,048 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:22:21,048 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened fbbc71fa787b8911356ef7975af46e80; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74906596, jitterRate=0.11619526147842407}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T02:22:21,048 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:21,049 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for fbbc71fa787b8911356ef7975af46e80: Running coprocessor pre-open hook at 1731118941032Writing region info on filesystem at 1731118941032Initializing all the Stores at 1731118941034 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118941034Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118941034Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118941034Cleaning up temporary data from old regions at 1731118941044 (+10 ms)Running coprocessor post-open hooks at 1731118941048 (+4 ms)Region opened successfully at 1731118941049 (+1 ms) 2024-11-09T02:22:21,049 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., pid=61, masterSystemTime=1731118941028 2024-11-09T02:22:21,051 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:21,051 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:21,051 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=fbbc71fa787b8911356ef7975af46e80, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:21,053 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 because future has completed 2024-11-09T02:22:21,055 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-09T02:22:21,055 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 in 177 msec 2024-11-09T02:22:21,057 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-09T02:22:21,057 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fbbc71fa787b8911356ef7975af46e80, ASSIGN in 335 msec 2024-11-09T02:22:21,057 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:22:21,058 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118941057"}]},"ts":"1731118941057"} 2024-11-09T02:22:21,059 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T02:22:21,059 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:22:21,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1790 sec 2024-11-09T02:22:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-09T02:22:22,041 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:22:22,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,043 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:22:22,043 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:22:22,044 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:22:22,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:22:22,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:22:22,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e64169b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:22:22,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:22:22,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:22,046 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:22:22,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6903cc85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,047 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:22:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:22:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:22:22,049 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:22:22,050 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T02:22:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 53f286d3af3b,45041,1731118874457 2024-11-09T02:22:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@11457d73 2024-11-09T02:22:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T02:22:22,052 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T02:22:22,054 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,054 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:22:22,054 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:22:22,054 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:22:22,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:22:22,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:22:22,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9d939c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:22:22,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:22:22,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:22,057 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:22:22,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@768b647e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,058 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,058 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:22:22,058 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:22:22,059 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:22:22,059 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:22:22,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:22:22,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@967eb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:22:22,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:22:22,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:22,061 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:22:22,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ed2d3f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,062 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,062 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:22:22,062 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:22:22,063 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:22:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:22:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:22:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3072b4af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:22:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:22:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:22,064 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:22:22,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f7a9592, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,065 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,065 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:22:22,065 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:22:22,066 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:22:22,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:22:22,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:22:22,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4db5a411, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:22:22,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:22:22,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:22,067 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:22:22,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d2d330f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,069 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,069 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:22:22,069 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:22:22,069 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:22:22,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:22:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:22:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62c4e022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:22:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:22:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:22,071 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:22:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bf4cda3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,072 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:22:22,072 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:22:22,073 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:22:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:22:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:22:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eccbfc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:22:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:22:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:22,074 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:22:22,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d2ef8b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,075 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,075 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:22:22,075 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:22:22,076 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:22:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:22:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:22:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dea68df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:22:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:22:22,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:22,077 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:22:22,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4443fa6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,078 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,078 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:22:22,079 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:22:22,079 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:22:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:22:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:22:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24353083, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:22:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:22:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:22,080 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:22:22,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@194c4a76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,081 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,081 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:22:22,081 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:22:22,082 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:22:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:22:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:22:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6661b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:22:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:22:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:22,083 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:22:22,083 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a7521cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,084 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,084 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:22:22,084 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:22:22,085 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:22:22,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:22:22,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:22:22,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4090beac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:22:22,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:22:22,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:22,086 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:22:22,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@351494cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:22,087 DEBUG [Thread-1126 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:22:22,087 DEBUG [Thread-1111 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:22:22,087 DEBUG [Thread-1115 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:22:22,087 DEBUG [Thread-1120 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:22:22,087 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:22:22,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:22:22,087 DEBUG [Thread-1118 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:22:22,087 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:22:22,087 DEBUG [Thread-1107 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:22:22,088 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:22:22,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:22:22,088 DEBUG [Thread-1122 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:22:22,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:22:22,088 DEBUG [Thread-1109 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:22:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-09T02:22:22,088 DEBUG [Thread-1124 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:22:22,088 DEBUG [Thread-1122 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:22:22,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:22:22,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:22:22,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:22:22,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:22:22,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:22:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T02:22:22,089 DEBUG [Thread-1122 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:22:22,089 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:22:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:22:22,090 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:22:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:22:22,090 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:22:22,090 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:22:22,090 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:22:22,090 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:22:22,090 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:22:22,090 DEBUG [Thread-1107 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:22:22,091 DEBUG [Thread-1107 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:22:22,091 DEBUG [Thread-1124 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:22:22,091 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:22:22,091 DEBUG [Thread-1124 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:22:22,091 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:22:22,091 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:22:22,092 DEBUG [Thread-1113 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:22:22,092 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:22:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:22:22,092 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:22:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:22:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:22:22,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:22:22,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:22:22,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:22:22,094 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:22:22,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:22:22,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:22:22,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:22:22,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:22:22,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:22:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:22:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:22,105 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:22:22,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbbc71fa787b8911356ef7975af46e80 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T02:22:22,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/8814a5292faa4a55a942ce48c172b3bd is 50, key is test_row_0/A:col10/1731118942098/Put/seqid=0 2024-11-09T02:22:22,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742062_1238 (size=12001) 2024-11-09T02:22:22,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119002136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:22,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:22,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119002138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:22,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:22,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119002139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119002140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:22,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:22,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:22,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119002141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:22,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:22,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T02:22:22,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-09T02:22:22,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:22,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:22,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:22,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:22,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:22,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119002250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:22,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:22,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119002250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:22,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:22,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119002250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:22,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119002250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:22,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:22,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:22,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119002250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:22,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:22,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-09T02:22:22,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:22,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:22,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:22,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:22,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T02:22:22,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119002460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:22,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:22,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119002460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:22,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119002460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:22,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119002460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119002461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:22,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/8814a5292faa4a55a942ce48c172b3bd 2024-11-09T02:22:22,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-09T02:22:22,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:22,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:22,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:22,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:22,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:22,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/f46c9f5291da47a0bc4e475a0400817d is 50, key is test_row_0/B:col10/1731118942098/Put/seqid=0 2024-11-09T02:22:22,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742063_1239 (size=12001) 2024-11-09T02:22:22,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/f46c9f5291da47a0bc4e475a0400817d 2024-11-09T02:22:22,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/9b216fc7c2d748bca9b97c624fd23526 is 50, key is test_row_0/C:col10/1731118942098/Put/seqid=0 2024-11-09T02:22:22,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742064_1240 (size=12001) 2024-11-09T02:22:22,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-09T02:22:22,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:22,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:22,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:22,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:22,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T02:22:22,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119002770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:22,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:22,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119002770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119002770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:22,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119002770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119002770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:22,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:22,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:22,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-09T02:22:22,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:22,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:22,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:22,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:22,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:23,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-09T02:22:23,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:23,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:23,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:23,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:23,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:23,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/9b216fc7c2d748bca9b97c624fd23526 2024-11-09T02:22:23,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/8814a5292faa4a55a942ce48c172b3bd as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/8814a5292faa4a55a942ce48c172b3bd 2024-11-09T02:22:23,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/8814a5292faa4a55a942ce48c172b3bd, entries=150, sequenceid=12, filesize=11.7 K 2024-11-09T02:22:23,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/f46c9f5291da47a0bc4e475a0400817d as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/f46c9f5291da47a0bc4e475a0400817d 2024-11-09T02:22:23,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/f46c9f5291da47a0bc4e475a0400817d, entries=150, sequenceid=12, filesize=11.7 K 2024-11-09T02:22:23,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/9b216fc7c2d748bca9b97c624fd23526 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/9b216fc7c2d748bca9b97c624fd23526 2024-11-09T02:22:23,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/9b216fc7c2d748bca9b97c624fd23526, entries=150, sequenceid=12, filesize=11.7 K 2024-11-09T02:22:23,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for fbbc71fa787b8911356ef7975af46e80 in 942ms, sequenceid=12, compaction requested=false 2024-11-09T02:22:23,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:23,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-09T02:22:23,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:23,166 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing fbbc71fa787b8911356ef7975af46e80 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T02:22:23,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/e8a708c72be34ba4a435eb74cb39279d is 50, key is test_row_0/A:col10/1731118942137/Put/seqid=0 2024-11-09T02:22:23,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742065_1241 (size=12001) 2024-11-09T02:22:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T02:22:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:23,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:23,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119003284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:23,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:23,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119003287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:23,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:23,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119003287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:23,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:23,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119003290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119003290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:23,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:23,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:23,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119003400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:23,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:23,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:23,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119003400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:23,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119003400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119003400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:23,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119003400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:23,575 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/e8a708c72be34ba4a435eb74cb39279d 2024-11-09T02:22:23,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/2eb5de7a6d80499ba384983328d5f7a5 is 50, key is test_row_0/B:col10/1731118942137/Put/seqid=0 2024-11-09T02:22:23,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742066_1242 (size=12001) 2024-11-09T02:22:23,607 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/2eb5de7a6d80499ba384983328d5f7a5 2024-11-09T02:22:23,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:23,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119003610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119003610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:23,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119003610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:23,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:23,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:23,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119003611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:23,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:23,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119003611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:23,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/d1f3b0e83a2343d2a8243cd47dd72e9b is 50, key is test_row_0/C:col10/1731118942137/Put/seqid=0 2024-11-09T02:22:23,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742067_1243 (size=12001) 2024-11-09T02:22:23,647 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/d1f3b0e83a2343d2a8243cd47dd72e9b 2024-11-09T02:22:23,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/e8a708c72be34ba4a435eb74cb39279d as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/e8a708c72be34ba4a435eb74cb39279d 2024-11-09T02:22:23,665 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/e8a708c72be34ba4a435eb74cb39279d, entries=150, sequenceid=37, filesize=11.7 K 2024-11-09T02:22:23,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/2eb5de7a6d80499ba384983328d5f7a5 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/2eb5de7a6d80499ba384983328d5f7a5 2024-11-09T02:22:23,691 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/2eb5de7a6d80499ba384983328d5f7a5, entries=150, sequenceid=37, filesize=11.7 K 2024-11-09T02:22:23,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/d1f3b0e83a2343d2a8243cd47dd72e9b as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/d1f3b0e83a2343d2a8243cd47dd72e9b 2024-11-09T02:22:23,706 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/d1f3b0e83a2343d2a8243cd47dd72e9b, entries=150, sequenceid=37, filesize=11.7 K 2024-11-09T02:22:23,707 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for fbbc71fa787b8911356ef7975af46e80 in 541ms, sequenceid=37, compaction requested=false 2024-11-09T02:22:23,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:23,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:23,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-09T02:22:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-09T02:22:23,715 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-09T02:22:23,715 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6180 sec 2024-11-09T02:22:23,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.6310 sec 2024-11-09T02:22:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:23,922 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:22:23,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbbc71fa787b8911356ef7975af46e80 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-09T02:22:23,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/6bd0a13da40c41f1b445eaddfc9f6348 is 50, key is test_row_0/A:col10/1731118943922/Put/seqid=0 2024-11-09T02:22:23,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742068_1244 (size=12001) 2024-11-09T02:22:23,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119003938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:23,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119003940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:23,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119003942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:23,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119003942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:23,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:23,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:23,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119003943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:23,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:23,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:24,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119004049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:24,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119004050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119004050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:24,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119004050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:24,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:24,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119004050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:24,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:24,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T02:22:24,240 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:22:24,240 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:22:24,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-09T02:22:24,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-09T02:22:24,242 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:22:24,242 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:22:24,242 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:22:24,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119004260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:24,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119004260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119004260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:24,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119004260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:24,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119004260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:24,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/6bd0a13da40c41f1b445eaddfc9f6348 2024-11-09T02:22:24,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/e4c6a2a731ff451e92afd84deb5dad8a is 50, key is test_row_0/B:col10/1731118943922/Put/seqid=0 2024-11-09T02:22:24,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742069_1245 (size=12001) 2024-11-09T02:22:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-09T02:22:24,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-09T02:22:24,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:24,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:24,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:24,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:24,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:24,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:24,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-09T02:22:24,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:24,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:24,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:24,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:24,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-09T02:22:24,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119004573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:24,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119004573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:24,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:24,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119004573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:24,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119004573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:24,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119004573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:24,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:24,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:24,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-09T02:22:24,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:24,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:24,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:24,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:24,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:24,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/e4c6a2a731ff451e92afd84deb5dad8a 2024-11-09T02:22:24,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/13b2310422094204a8cd9253620d4da4 is 50, key is test_row_0/C:col10/1731118943922/Put/seqid=0 2024-11-09T02:22:24,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742070_1246 (size=12001) 2024-11-09T02:22:24,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-09T02:22:24,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:24,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:24,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:24,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:24,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-09T02:22:25,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-09T02:22:25,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:25,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:25,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:25,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:25,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:25,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:25,078 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:22:25,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:25,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119005080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:25,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119005080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:25,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:25,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:25,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:25,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:25,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119005080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119005080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:25,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:25,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:25,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:25,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:25,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:25,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:25,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:25,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:25,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:25,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:25,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119005090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:25,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:25,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:25,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:25,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/13b2310422094204a8cd9253620d4da4 2024-11-09T02:22:25,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/6bd0a13da40c41f1b445eaddfc9f6348 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/6bd0a13da40c41f1b445eaddfc9f6348 2024-11-09T02:22:25,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-09T02:22:25,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:25,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:25,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:25,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:25,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:25,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/6bd0a13da40c41f1b445eaddfc9f6348, entries=150, sequenceid=52, filesize=11.7 K 2024-11-09T02:22:25,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:25,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/e4c6a2a731ff451e92afd84deb5dad8a as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/e4c6a2a731ff451e92afd84deb5dad8a 2024-11-09T02:22:25,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/e4c6a2a731ff451e92afd84deb5dad8a, entries=150, sequenceid=52, filesize=11.7 K 2024-11-09T02:22:25,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/13b2310422094204a8cd9253620d4da4 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/13b2310422094204a8cd9253620d4da4 2024-11-09T02:22:25,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/13b2310422094204a8cd9253620d4da4, entries=150, sequenceid=52, filesize=11.7 K 2024-11-09T02:22:25,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for fbbc71fa787b8911356ef7975af46e80 in 1251ms, sequenceid=52, compaction requested=true 2024-11-09T02:22:25,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:25,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbbc71fa787b8911356ef7975af46e80:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:22:25,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:25,173 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:25,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbbc71fa787b8911356ef7975af46e80:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:22:25,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:25,174 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:25,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbbc71fa787b8911356ef7975af46e80:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:22:25,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:25,174 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:25,175 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): fbbc71fa787b8911356ef7975af46e80/A is initiating minor compaction (all files) 2024-11-09T02:22:25,175 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbbc71fa787b8911356ef7975af46e80/A in TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:25,175 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:25,175 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/8814a5292faa4a55a942ce48c172b3bd, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/e8a708c72be34ba4a435eb74cb39279d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/6bd0a13da40c41f1b445eaddfc9f6348] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp, totalSize=35.2 K 2024-11-09T02:22:25,175 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): fbbc71fa787b8911356ef7975af46e80/B is initiating minor compaction (all files) 2024-11-09T02:22:25,175 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbbc71fa787b8911356ef7975af46e80/B in TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:25,175 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/f46c9f5291da47a0bc4e475a0400817d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/2eb5de7a6d80499ba384983328d5f7a5, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/e4c6a2a731ff451e92afd84deb5dad8a] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp, totalSize=35.2 K 2024-11-09T02:22:25,175 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8814a5292faa4a55a942ce48c172b3bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731118942098 2024-11-09T02:22:25,175 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting f46c9f5291da47a0bc4e475a0400817d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731118942098 2024-11-09T02:22:25,176 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8a708c72be34ba4a435eb74cb39279d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731118942137 2024-11-09T02:22:25,176 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 2eb5de7a6d80499ba384983328d5f7a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731118942137 2024-11-09T02:22:25,176 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting e4c6a2a731ff451e92afd84deb5dad8a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731118943287 2024-11-09T02:22:25,176 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bd0a13da40c41f1b445eaddfc9f6348, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731118943287 2024-11-09T02:22:25,199 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbbc71fa787b8911356ef7975af46e80#B#compaction#201 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:25,199 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbbc71fa787b8911356ef7975af46e80#A#compaction#202 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:25,199 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/594a8d05a8c9422da679bc1a1ac1ed5e is 50, key is test_row_0/B:col10/1731118943922/Put/seqid=0 2024-11-09T02:22:25,199 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/f05073feb95a42ffb3d18db3b74808f9 is 50, key is test_row_0/A:col10/1731118943922/Put/seqid=0 2024-11-09T02:22:25,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742072_1248 (size=12104) 2024-11-09T02:22:25,224 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/f05073feb95a42ffb3d18db3b74808f9 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/f05073feb95a42ffb3d18db3b74808f9 2024-11-09T02:22:25,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742071_1247 (size=12104) 2024-11-09T02:22:25,231 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbbc71fa787b8911356ef7975af46e80/A of fbbc71fa787b8911356ef7975af46e80 into f05073feb95a42ffb3d18db3b74808f9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:25,231 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:25,231 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., storeName=fbbc71fa787b8911356ef7975af46e80/A, priority=13, startTime=1731118945173; duration=0sec 2024-11-09T02:22:25,231 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:25,231 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbbc71fa787b8911356ef7975af46e80:A 2024-11-09T02:22:25,231 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:25,233 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/594a8d05a8c9422da679bc1a1ac1ed5e as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/594a8d05a8c9422da679bc1a1ac1ed5e 2024-11-09T02:22:25,233 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:25,233 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): fbbc71fa787b8911356ef7975af46e80/C is initiating minor compaction (all files) 2024-11-09T02:22:25,234 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbbc71fa787b8911356ef7975af46e80/C in TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:25,234 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/9b216fc7c2d748bca9b97c624fd23526, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/d1f3b0e83a2343d2a8243cd47dd72e9b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/13b2310422094204a8cd9253620d4da4] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp, totalSize=35.2 K 2024-11-09T02:22:25,235 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b216fc7c2d748bca9b97c624fd23526, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731118942098 2024-11-09T02:22:25,235 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1f3b0e83a2343d2a8243cd47dd72e9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731118942137 2024-11-09T02:22:25,236 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13b2310422094204a8cd9253620d4da4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731118943287 2024-11-09T02:22:25,240 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbbc71fa787b8911356ef7975af46e80/B of fbbc71fa787b8911356ef7975af46e80 into 594a8d05a8c9422da679bc1a1ac1ed5e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:25,240 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:25,240 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., storeName=fbbc71fa787b8911356ef7975af46e80/B, priority=13, startTime=1731118945173; duration=0sec 2024-11-09T02:22:25,240 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:25,240 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbbc71fa787b8911356ef7975af46e80:B 2024-11-09T02:22:25,246 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbbc71fa787b8911356ef7975af46e80#C#compaction#203 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:25,247 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/59dde14b2dd444438283a11590030afb is 50, key is test_row_0/C:col10/1731118943922/Put/seqid=0 2024-11-09T02:22:25,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742073_1249 (size=12104) 2024-11-09T02:22:25,258 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/59dde14b2dd444438283a11590030afb as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/59dde14b2dd444438283a11590030afb 2024-11-09T02:22:25,265 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbbc71fa787b8911356ef7975af46e80/C of fbbc71fa787b8911356ef7975af46e80 into 59dde14b2dd444438283a11590030afb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:25,265 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:25,265 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., storeName=fbbc71fa787b8911356ef7975af46e80/C, priority=13, startTime=1731118945174; duration=0sec 2024-11-09T02:22:25,265 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:25,265 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbbc71fa787b8911356ef7975af46e80:C 2024-11-09T02:22:25,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-09T02:22:25,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:25,315 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing fbbc71fa787b8911356ef7975af46e80 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-09T02:22:25,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/9b7442e7398e48ecada748063fe639bc is 50, key is test_row_0/A:col10/1731118943941/Put/seqid=0 2024-11-09T02:22:25,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742074_1250 (size=12001) 2024-11-09T02:22:25,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-09T02:22:25,773 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/9b7442e7398e48ecada748063fe639bc 2024-11-09T02:22:25,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/e245253a16794653a06babc2bf06d229 is 50, key is test_row_0/B:col10/1731118943941/Put/seqid=0 2024-11-09T02:22:25,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742075_1251 (size=12001) 2024-11-09T02:22:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:26,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:26,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119006102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:26,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:26,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119006102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:26,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:26,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119006103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:26,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119006104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:26,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119006105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:26,196 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/e245253a16794653a06babc2bf06d229 2024-11-09T02:22:26,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/97e5029bb72e428a80bae35904d61568 is 50, key is test_row_0/C:col10/1731118943941/Put/seqid=0 2024-11-09T02:22:26,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119006209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:26,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:26,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119006210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:26,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:26,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119006210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:26,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:26,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:26,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119006210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:26,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119006210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:26,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742076_1252 (size=12001) 2024-11-09T02:22:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-09T02:22:26,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:26,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119006420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119006420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:26,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:26,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:26,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119006420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119006420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:26,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:26,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119006421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:26,622 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/97e5029bb72e428a80bae35904d61568 2024-11-09T02:22:26,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/9b7442e7398e48ecada748063fe639bc as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/9b7442e7398e48ecada748063fe639bc 2024-11-09T02:22:26,631 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/9b7442e7398e48ecada748063fe639bc, entries=150, sequenceid=76, filesize=11.7 K 2024-11-09T02:22:26,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/e245253a16794653a06babc2bf06d229 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/e245253a16794653a06babc2bf06d229 2024-11-09T02:22:26,636 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/e245253a16794653a06babc2bf06d229, entries=150, sequenceid=76, filesize=11.7 K 2024-11-09T02:22:26,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/97e5029bb72e428a80bae35904d61568 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/97e5029bb72e428a80bae35904d61568 2024-11-09T02:22:26,642 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/97e5029bb72e428a80bae35904d61568, entries=150, sequenceid=76, filesize=11.7 K 2024-11-09T02:22:26,643 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for fbbc71fa787b8911356ef7975af46e80 in 1328ms, sequenceid=76, compaction requested=false 2024-11-09T02:22:26,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:26,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:26,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-09T02:22:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-09T02:22:26,646 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-09T02:22:26,646 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4030 sec 2024-11-09T02:22:26,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.4070 sec 2024-11-09T02:22:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:26,732 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:22:26,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbbc71fa787b8911356ef7975af46e80 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-09T02:22:26,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/2e9fa72cfa4346dd9520d19a9e72dcf8 is 50, key is test_row_0/A:col10/1731118946731/Put/seqid=0 2024-11-09T02:22:26,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742077_1253 (size=12001) 2024-11-09T02:22:26,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119006748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:26,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:26,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119006750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:26,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:26,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119006750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:26,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:26,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119006751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:26,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:26,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:26,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119006754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:26,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:26,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119006860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:26,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119006860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:26,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119006860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:26,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:26,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119006860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:26,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:26,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119006860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:26,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:26,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:27,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119007070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:27,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:27,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119007070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:27,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:27,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119007070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:27,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:27,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:27,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119007070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119007070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:27,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:27,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:27,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/2e9fa72cfa4346dd9520d19a9e72dcf8 2024-11-09T02:22:27,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/95f3fc3376dd4da19c7c3711d3fdeb1d is 50, key is test_row_0/B:col10/1731118946731/Put/seqid=0 2024-11-09T02:22:27,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742078_1254 (size=12001) 2024-11-09T02:22:27,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/95f3fc3376dd4da19c7c3711d3fdeb1d 2024-11-09T02:22:27,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/17bf9b4c2bc44c2b971bc1dc4acbbf0e is 50, key is test_row_0/C:col10/1731118946731/Put/seqid=0 2024-11-09T02:22:27,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742079_1255 (size=12001) 2024-11-09T02:22:27,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119007380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:27,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:27,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119007381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:27,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:27,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:27,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119007381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:27,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:27,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119007381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:27,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:27,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119007381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:27,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:27,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:27,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/17bf9b4c2bc44c2b971bc1dc4acbbf0e 2024-11-09T02:22:27,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/2e9fa72cfa4346dd9520d19a9e72dcf8 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/2e9fa72cfa4346dd9520d19a9e72dcf8 2024-11-09T02:22:27,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/2e9fa72cfa4346dd9520d19a9e72dcf8, entries=150, sequenceid=95, filesize=11.7 K 2024-11-09T02:22:27,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/95f3fc3376dd4da19c7c3711d3fdeb1d as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/95f3fc3376dd4da19c7c3711d3fdeb1d 2024-11-09T02:22:27,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/95f3fc3376dd4da19c7c3711d3fdeb1d, entries=150, sequenceid=95, filesize=11.7 K 2024-11-09T02:22:27,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/17bf9b4c2bc44c2b971bc1dc4acbbf0e as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/17bf9b4c2bc44c2b971bc1dc4acbbf0e 2024-11-09T02:22:27,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/17bf9b4c2bc44c2b971bc1dc4acbbf0e, entries=150, sequenceid=95, filesize=11.7 K 2024-11-09T02:22:27,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for fbbc71fa787b8911356ef7975af46e80 in 915ms, sequenceid=95, compaction requested=true 2024-11-09T02:22:27,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:27,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbbc71fa787b8911356ef7975af46e80:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:22:27,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:27,647 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:27,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbbc71fa787b8911356ef7975af46e80:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:22:27,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:27,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbbc71fa787b8911356ef7975af46e80:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:22:27,647 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:27,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:27,648 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:27,648 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:27,648 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): fbbc71fa787b8911356ef7975af46e80/B is initiating minor compaction (all files) 2024-11-09T02:22:27,648 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): fbbc71fa787b8911356ef7975af46e80/A is initiating minor compaction (all files) 2024-11-09T02:22:27,648 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbbc71fa787b8911356ef7975af46e80/B in TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:27,649 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/594a8d05a8c9422da679bc1a1ac1ed5e, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/e245253a16794653a06babc2bf06d229, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/95f3fc3376dd4da19c7c3711d3fdeb1d] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp, totalSize=35.3 K 2024-11-09T02:22:27,649 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbbc71fa787b8911356ef7975af46e80/A in TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:27,649 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/f05073feb95a42ffb3d18db3b74808f9, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/9b7442e7398e48ecada748063fe639bc, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/2e9fa72cfa4346dd9520d19a9e72dcf8] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp, totalSize=35.3 K 2024-11-09T02:22:27,649 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting f05073feb95a42ffb3d18db3b74808f9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731118943287 2024-11-09T02:22:27,649 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 594a8d05a8c9422da679bc1a1ac1ed5e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731118943287 2024-11-09T02:22:27,650 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting e245253a16794653a06babc2bf06d229, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731118943940 2024-11-09T02:22:27,650 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b7442e7398e48ecada748063fe639bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731118943940 2024-11-09T02:22:27,650 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 95f3fc3376dd4da19c7c3711d3fdeb1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731118946731 2024-11-09T02:22:27,650 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e9fa72cfa4346dd9520d19a9e72dcf8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731118946731 2024-11-09T02:22:27,670 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbbc71fa787b8911356ef7975af46e80#B#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:27,670 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/c55374cde69a4942a9cd105c33329fab is 50, key is test_row_0/B:col10/1731118946731/Put/seqid=0 2024-11-09T02:22:27,672 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbbc71fa787b8911356ef7975af46e80#A#compaction#211 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:27,673 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/073bf17c05054caeaeee8549b6a2d999 is 50, key is test_row_0/A:col10/1731118946731/Put/seqid=0 2024-11-09T02:22:27,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742080_1256 (size=12207) 2024-11-09T02:22:27,688 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/c55374cde69a4942a9cd105c33329fab as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/c55374cde69a4942a9cd105c33329fab 2024-11-09T02:22:27,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742081_1257 (size=12207) 2024-11-09T02:22:27,696 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbbc71fa787b8911356ef7975af46e80/B of fbbc71fa787b8911356ef7975af46e80 into c55374cde69a4942a9cd105c33329fab(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:27,696 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:27,696 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., storeName=fbbc71fa787b8911356ef7975af46e80/B, priority=13, startTime=1731118947647; duration=0sec 2024-11-09T02:22:27,696 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:27,696 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbbc71fa787b8911356ef7975af46e80:B 2024-11-09T02:22:27,697 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:27,698 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:27,698 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): fbbc71fa787b8911356ef7975af46e80/C is initiating minor compaction (all files) 2024-11-09T02:22:27,698 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbbc71fa787b8911356ef7975af46e80/C in TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:27,698 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/59dde14b2dd444438283a11590030afb, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/97e5029bb72e428a80bae35904d61568, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/17bf9b4c2bc44c2b971bc1dc4acbbf0e] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp, totalSize=35.3 K 2024-11-09T02:22:27,698 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 59dde14b2dd444438283a11590030afb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731118943287 2024-11-09T02:22:27,699 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 97e5029bb72e428a80bae35904d61568, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731118943940 2024-11-09T02:22:27,699 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 17bf9b4c2bc44c2b971bc1dc4acbbf0e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731118946731 2024-11-09T02:22:27,708 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbbc71fa787b8911356ef7975af46e80#C#compaction#212 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:27,709 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/2510b59bbd054c6a847fadd3c44c2520 is 50, key is test_row_0/C:col10/1731118946731/Put/seqid=0 2024-11-09T02:22:27,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742082_1258 (size=12207) 2024-11-09T02:22:27,721 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/2510b59bbd054c6a847fadd3c44c2520 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/2510b59bbd054c6a847fadd3c44c2520 2024-11-09T02:22:27,726 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbbc71fa787b8911356ef7975af46e80/C of fbbc71fa787b8911356ef7975af46e80 into 2510b59bbd054c6a847fadd3c44c2520(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:27,726 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:27,726 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., storeName=fbbc71fa787b8911356ef7975af46e80/C, priority=13, startTime=1731118947647; duration=0sec 2024-11-09T02:22:27,727 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:27,727 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbbc71fa787b8911356ef7975af46e80:C 2024-11-09T02:22:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:27,892 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:22:27,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbbc71fa787b8911356ef7975af46e80 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-09T02:22:27,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/621fbf339bb9461d9a3eeafba6dcc493 is 50, key is test_row_0/A:col10/1731118946740/Put/seqid=0 2024-11-09T02:22:27,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742083_1259 (size=12001) 2024-11-09T02:22:27,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119007907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:27,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119007907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:27,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119007911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:27,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119007913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:27,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119007913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:28,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119008020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:28,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119008020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:28,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:28,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:28,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119008020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:28,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:28,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119008020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:28,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:28,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119008030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:28,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:28,099 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/073bf17c05054caeaeee8549b6a2d999 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/073bf17c05054caeaeee8549b6a2d999 2024-11-09T02:22:28,103 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbbc71fa787b8911356ef7975af46e80/A of fbbc71fa787b8911356ef7975af46e80 into 073bf17c05054caeaeee8549b6a2d999(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:28,103 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:28,103 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., storeName=fbbc71fa787b8911356ef7975af46e80/A, priority=13, startTime=1731118947647; duration=0sec 2024-11-09T02:22:28,103 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:28,103 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbbc71fa787b8911356ef7975af46e80:A 2024-11-09T02:22:28,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:28,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119008230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:28,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:28,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119008230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:28,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:28,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:28,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:28,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119008230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:28,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:28,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119008230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:28,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:28,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:28,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:28,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119008240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:28,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:28,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/621fbf339bb9461d9a3eeafba6dcc493 2024-11-09T02:22:28,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/2ae5c6325d0640678582259c0da56ef7 is 50, key is test_row_0/B:col10/1731118946740/Put/seqid=0 2024-11-09T02:22:28,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742084_1260 (size=12001) 2024-11-09T02:22:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-09T02:22:28,410 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:22:28,410 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:22:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-09T02:22:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T02:22:28,415 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:22:28,415 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:22:28,416 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:22:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T02:22:28,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119008540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:28,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119008540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:28,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119008540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119008540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:28,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119008549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:28,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:28,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:28,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-09T02:22:28,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:28,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:28,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:28,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:28,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:28,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/2ae5c6325d0640678582259c0da56ef7 2024-11-09T02:22:28,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-09T02:22:28,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:28,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:28,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:28,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:28,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:28,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/9d6536d000754a5a964a9b0b1381ae6b is 50, key is test_row_0/C:col10/1731118946740/Put/seqid=0 2024-11-09T02:22:28,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742085_1261 (size=12001) 2024-11-09T02:22:28,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/9d6536d000754a5a964a9b0b1381ae6b 2024-11-09T02:22:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T02:22:28,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/621fbf339bb9461d9a3eeafba6dcc493 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/621fbf339bb9461d9a3eeafba6dcc493 2024-11-09T02:22:28,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/621fbf339bb9461d9a3eeafba6dcc493, entries=150, sequenceid=116, filesize=11.7 K 2024-11-09T02:22:28,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/2ae5c6325d0640678582259c0da56ef7 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/2ae5c6325d0640678582259c0da56ef7 2024-11-09T02:22:28,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/2ae5c6325d0640678582259c0da56ef7, entries=150, sequenceid=116, filesize=11.7 K 2024-11-09T02:22:28,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/9d6536d000754a5a964a9b0b1381ae6b as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/9d6536d000754a5a964a9b0b1381ae6b 2024-11-09T02:22:28,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/9d6536d000754a5a964a9b0b1381ae6b, entries=150, sequenceid=116, filesize=11.7 K 2024-11-09T02:22:28,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for fbbc71fa787b8911356ef7975af46e80 in 876ms, sequenceid=116, compaction requested=false 2024-11-09T02:22:28,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:28,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-09T02:22:28,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:28,873 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing fbbc71fa787b8911356ef7975af46e80 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-09T02:22:28,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/43649e5d0dae417aaa612e9c34b62b43 is 50, key is test_row_0/A:col10/1731118947910/Put/seqid=0 2024-11-09T02:22:28,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742086_1262 (size=12151) 2024-11-09T02:22:28,899 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/43649e5d0dae417aaa612e9c34b62b43 2024-11-09T02:22:28,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/46176b8aed084053a738ac161ae73063 is 50, key is test_row_0/B:col10/1731118947910/Put/seqid=0 2024-11-09T02:22:28,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742087_1263 (size=12151) 2024-11-09T02:22:28,926 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/46176b8aed084053a738ac161ae73063 2024-11-09T02:22:28,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/479b4f88c8fa487fb1561779294420eb is 50, key is test_row_0/C:col10/1731118947910/Put/seqid=0 2024-11-09T02:22:28,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742088_1264 (size=12151) 2024-11-09T02:22:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T02:22:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:29,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:29,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119009063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:29,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:29,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119009064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:29,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:29,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119009065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:29,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119009066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:29,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:29,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119009067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:29,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:29,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119009170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:29,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:29,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119009180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:29,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119009180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:29,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119009180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:29,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119009180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:29,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:29,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:29,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:29,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:29,349 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/479b4f88c8fa487fb1561779294420eb 2024-11-09T02:22:29,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/43649e5d0dae417aaa612e9c34b62b43 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/43649e5d0dae417aaa612e9c34b62b43 2024-11-09T02:22:29,358 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/43649e5d0dae417aaa612e9c34b62b43, entries=150, sequenceid=135, filesize=11.9 K 2024-11-09T02:22:29,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/46176b8aed084053a738ac161ae73063 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/46176b8aed084053a738ac161ae73063 2024-11-09T02:22:29,363 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/46176b8aed084053a738ac161ae73063, entries=150, sequenceid=135, filesize=11.9 K 2024-11-09T02:22:29,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/479b4f88c8fa487fb1561779294420eb as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/479b4f88c8fa487fb1561779294420eb 2024-11-09T02:22:29,369 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/479b4f88c8fa487fb1561779294420eb, entries=150, sequenceid=135, filesize=11.9 K 2024-11-09T02:22:29,370 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for fbbc71fa787b8911356ef7975af46e80 in 497ms, sequenceid=135, compaction requested=true 2024-11-09T02:22:29,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:29,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:29,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-09T02:22:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-09T02:22:29,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-09T02:22:29,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 956 msec 2024-11-09T02:22:29,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 962 msec 2024-11-09T02:22:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:29,383 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:22:29,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbbc71fa787b8911356ef7975af46e80 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-09T02:22:29,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/79b2f430b94d453186e0aa7942ba0940 is 50, key is test_row_0/A:col10/1731118949063/Put/seqid=0 2024-11-09T02:22:29,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742089_1265 (size=14541) 2024-11-09T02:22:29,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119009401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:29,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:29,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119009402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:29,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:29,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119009403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:29,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:29,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/79b2f430b94d453186e0aa7942ba0940 2024-11-09T02:22:29,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:29,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119009405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:29,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:29,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119009406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:29,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:29,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/dd85ba50eeba46efb9e9354fbc47db57 is 50, key is test_row_0/B:col10/1731118949063/Put/seqid=0 2024-11-09T02:22:29,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742090_1266 (size=12151) 2024-11-09T02:22:29,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:29,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119009509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119009509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:29,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:29,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:29,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119009520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:29,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119009520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119009520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T02:22:29,550 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:22:29,550 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:22:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-09T02:22:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T02:22:29,552 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:22:29,552 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:22:29,552 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:22:29,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T02:22:29,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T02:22:29,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:29,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:29,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:29,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:29,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:29,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119009720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:29,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:29,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119009720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:29,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:29,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:29,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119009730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119009730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:29,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:29,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:29,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119009730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:29,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:29,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:29,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/dd85ba50eeba46efb9e9354fbc47db57 2024-11-09T02:22:29,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/f2c4399a85774b16badc9dcb4039a384 is 50, key is test_row_0/C:col10/1731118949063/Put/seqid=0 2024-11-09T02:22:29,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742091_1267 (size=12151) 2024-11-09T02:22:29,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T02:22:29,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:29,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:29,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:29,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:29,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:29,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:29,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T02:22:30,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T02:22:30,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:30,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:30,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:30,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:30,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:30,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:30,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:30,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119010030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:30,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119010030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:30,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:30,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119010040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:30,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:30,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119010040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:30,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:30,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:30,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119010040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:30,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:30,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:30,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T02:22:30,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:30,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:30,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:30,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:30,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:30,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:30,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T02:22:30,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/f2c4399a85774b16badc9dcb4039a384 2024-11-09T02:22:30,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/79b2f430b94d453186e0aa7942ba0940 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/79b2f430b94d453186e0aa7942ba0940 2024-11-09T02:22:30,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/79b2f430b94d453186e0aa7942ba0940, entries=200, sequenceid=154, filesize=14.2 K 2024-11-09T02:22:30,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/dd85ba50eeba46efb9e9354fbc47db57 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/dd85ba50eeba46efb9e9354fbc47db57 2024-11-09T02:22:30,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/dd85ba50eeba46efb9e9354fbc47db57, entries=150, sequenceid=154, filesize=11.9 K 2024-11-09T02:22:30,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/f2c4399a85774b16badc9dcb4039a384 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/f2c4399a85774b16badc9dcb4039a384 2024-11-09T02:22:30,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/f2c4399a85774b16badc9dcb4039a384, entries=150, sequenceid=154, filesize=11.9 K 2024-11-09T02:22:30,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for fbbc71fa787b8911356ef7975af46e80 in 885ms, sequenceid=154, compaction requested=true 2024-11-09T02:22:30,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:30,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbbc71fa787b8911356ef7975af46e80:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:22:30,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:30,269 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T02:22:30,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbbc71fa787b8911356ef7975af46e80:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:22:30,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:30,269 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T02:22:30,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbbc71fa787b8911356ef7975af46e80:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:22:30,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:30,270 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50900 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T02:22:30,270 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T02:22:30,270 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): fbbc71fa787b8911356ef7975af46e80/A is initiating minor compaction (all files) 2024-11-09T02:22:30,270 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): fbbc71fa787b8911356ef7975af46e80/B is initiating minor compaction (all files) 2024-11-09T02:22:30,270 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbbc71fa787b8911356ef7975af46e80/A in TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:30,270 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbbc71fa787b8911356ef7975af46e80/B in TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:30,270 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/073bf17c05054caeaeee8549b6a2d999, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/621fbf339bb9461d9a3eeafba6dcc493, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/43649e5d0dae417aaa612e9c34b62b43, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/79b2f430b94d453186e0aa7942ba0940] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp, totalSize=49.7 K 2024-11-09T02:22:30,270 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/c55374cde69a4942a9cd105c33329fab, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/2ae5c6325d0640678582259c0da56ef7, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/46176b8aed084053a738ac161ae73063, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/dd85ba50eeba46efb9e9354fbc47db57] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp, totalSize=47.4 K 2024-11-09T02:22:30,271 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 073bf17c05054caeaeee8549b6a2d999, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731118946731 2024-11-09T02:22:30,271 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting c55374cde69a4942a9cd105c33329fab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731118946731 2024-11-09T02:22:30,271 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 621fbf339bb9461d9a3eeafba6dcc493, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731118946740 2024-11-09T02:22:30,271 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ae5c6325d0640678582259c0da56ef7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731118946740 2024-11-09T02:22:30,272 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43649e5d0dae417aaa612e9c34b62b43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731118947906 2024-11-09T02:22:30,272 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 46176b8aed084053a738ac161ae73063, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731118947906 2024-11-09T02:22:30,272 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting dd85ba50eeba46efb9e9354fbc47db57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731118949063 2024-11-09T02:22:30,273 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79b2f430b94d453186e0aa7942ba0940, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731118949063 2024-11-09T02:22:30,282 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbbc71fa787b8911356ef7975af46e80#B#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:30,282 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbbc71fa787b8911356ef7975af46e80#A#compaction#223 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:30,282 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/cdca675112a24502ad3533328959efbf is 50, key is test_row_0/B:col10/1731118949063/Put/seqid=0 2024-11-09T02:22:30,282 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/f7ceb3108a6145ff9ac13e043bf5dd96 is 50, key is test_row_0/A:col10/1731118949063/Put/seqid=0 2024-11-09T02:22:30,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742092_1268 (size=12493) 2024-11-09T02:22:30,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742093_1269 (size=12493) 2024-11-09T02:22:30,308 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/cdca675112a24502ad3533328959efbf as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/cdca675112a24502ad3533328959efbf 2024-11-09T02:22:30,314 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fbbc71fa787b8911356ef7975af46e80/B of fbbc71fa787b8911356ef7975af46e80 into cdca675112a24502ad3533328959efbf(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:30,314 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:30,314 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., storeName=fbbc71fa787b8911356ef7975af46e80/B, priority=12, startTime=1731118950269; duration=0sec 2024-11-09T02:22:30,314 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:30,314 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbbc71fa787b8911356ef7975af46e80:B 2024-11-09T02:22:30,314 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T02:22:30,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T02:22:30,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:30,316 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing fbbc71fa787b8911356ef7975af46e80 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-09T02:22:30,316 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T02:22:30,316 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): fbbc71fa787b8911356ef7975af46e80/C is initiating minor compaction (all files) 2024-11-09T02:22:30,316 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbbc71fa787b8911356ef7975af46e80/C in TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:30,317 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/2510b59bbd054c6a847fadd3c44c2520, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/9d6536d000754a5a964a9b0b1381ae6b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/479b4f88c8fa487fb1561779294420eb, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/f2c4399a85774b16badc9dcb4039a384] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp, totalSize=47.4 K 2024-11-09T02:22:30,317 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 2510b59bbd054c6a847fadd3c44c2520, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731118946731 2024-11-09T02:22:30,317 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d6536d000754a5a964a9b0b1381ae6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731118946740 2024-11-09T02:22:30,318 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 479b4f88c8fa487fb1561779294420eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731118947906 2024-11-09T02:22:30,318 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting f2c4399a85774b16badc9dcb4039a384, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731118949063 2024-11-09T02:22:30,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/bc8ddf3498ad4b148d31470c4d67d016 is 50, key is test_row_0/A:col10/1731118949402/Put/seqid=0 2024-11-09T02:22:30,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742094_1270 (size=12151) 2024-11-09T02:22:30,329 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbbc71fa787b8911356ef7975af46e80#C#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:30,330 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/2824c221be86420798fb4660847d06ba is 50, key is test_row_0/C:col10/1731118949063/Put/seqid=0 2024-11-09T02:22:30,330 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/bc8ddf3498ad4b148d31470c4d67d016 2024-11-09T02:22:30,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742095_1271 (size=12493) 2024-11-09T02:22:30,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/26f5309c4a0e47469cd29bb924db9f96 is 50, key is test_row_0/B:col10/1731118949402/Put/seqid=0 2024-11-09T02:22:30,362 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/2824c221be86420798fb4660847d06ba as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/2824c221be86420798fb4660847d06ba 2024-11-09T02:22:30,370 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fbbc71fa787b8911356ef7975af46e80/C of fbbc71fa787b8911356ef7975af46e80 into 2824c221be86420798fb4660847d06ba(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:30,370 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:30,370 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., storeName=fbbc71fa787b8911356ef7975af46e80/C, priority=12, startTime=1731118950269; duration=0sec 2024-11-09T02:22:30,370 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:30,370 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbbc71fa787b8911356ef7975af46e80:C 2024-11-09T02:22:30,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742096_1272 (size=12151) 2024-11-09T02:22:30,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:30,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:30,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:30,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119010557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:30,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:30,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:30,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119010560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:30,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:30,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:30,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119010561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:30,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:30,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119010562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:30,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:30,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119010563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:30,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:30,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119010670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:30,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119010670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119010671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:30,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:30,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119010671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:30,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119010671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:30,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T02:22:30,707 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/f7ceb3108a6145ff9ac13e043bf5dd96 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/f7ceb3108a6145ff9ac13e043bf5dd96 2024-11-09T02:22:30,712 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fbbc71fa787b8911356ef7975af46e80/A of fbbc71fa787b8911356ef7975af46e80 into f7ceb3108a6145ff9ac13e043bf5dd96(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:30,712 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:30,712 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., storeName=fbbc71fa787b8911356ef7975af46e80/A, priority=12, startTime=1731118950269; duration=0sec 2024-11-09T02:22:30,712 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:30,712 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbbc71fa787b8911356ef7975af46e80:A 2024-11-09T02:22:30,773 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/26f5309c4a0e47469cd29bb924db9f96 2024-11-09T02:22:30,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/e02215022b9d4ebda06e4fa371fac865 is 50, key is test_row_0/C:col10/1731118949402/Put/seqid=0 2024-11-09T02:22:30,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742097_1273 (size=12151) 2024-11-09T02:22:30,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:30,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119010879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:30,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:30,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119010879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:30,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:30,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:30,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119010880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119010880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:30,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:30,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:30,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119010880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:30,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:30,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:30,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:31,189 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/e02215022b9d4ebda06e4fa371fac865 2024-11-09T02:22:31,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119011190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:31,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119011190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:31,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119011190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:31,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119011190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:31,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119011191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:31,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/bc8ddf3498ad4b148d31470c4d67d016 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/bc8ddf3498ad4b148d31470c4d67d016 2024-11-09T02:22:31,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:31,200 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/bc8ddf3498ad4b148d31470c4d67d016, entries=150, sequenceid=172, filesize=11.9 K 2024-11-09T02:22:31,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/26f5309c4a0e47469cd29bb924db9f96 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/26f5309c4a0e47469cd29bb924db9f96 2024-11-09T02:22:31,205 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/26f5309c4a0e47469cd29bb924db9f96, entries=150, sequenceid=172, filesize=11.9 K 2024-11-09T02:22:31,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/e02215022b9d4ebda06e4fa371fac865 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/e02215022b9d4ebda06e4fa371fac865 2024-11-09T02:22:31,212 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/e02215022b9d4ebda06e4fa371fac865, entries=150, sequenceid=172, filesize=11.9 K 2024-11-09T02:22:31,213 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for fbbc71fa787b8911356ef7975af46e80 in 897ms, sequenceid=172, compaction requested=false 2024-11-09T02:22:31,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:31,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:31,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-09T02:22:31,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-09T02:22:31,217 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-09T02:22:31,217 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6630 sec 2024-11-09T02:22:31,218 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.6670 sec 2024-11-09T02:22:31,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T02:22:31,700 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:22:31,701 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:22:31,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-09T02:22:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T02:22:31,703 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:22:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:31,703 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:22:31,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbbc71fa787b8911356ef7975af46e80 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-09T02:22:31,704 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:22:31,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:22:31,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/4d50155eb3d345d6b7160c3ede169655 is 50, key is test_row_0/A:col10/1731118951702/Put/seqid=0 2024-11-09T02:22:31,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742098_1274 (size=12151) 2024-11-09T02:22:31,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119011719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:31,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:31,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119011720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:31,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:31,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:31,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119011724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119011724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:31,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:31,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/4d50155eb3d345d6b7160c3ede169655 2024-11-09T02:22:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119011725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:31,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/41757ec5d1cf4951b6b9fb6d45231dbc is 50, key is test_row_0/B:col10/1731118951702/Put/seqid=0 2024-11-09T02:22:31,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742099_1275 (size=12151) 2024-11-09T02:22:31,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T02:22:31,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:31,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119011830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:31,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:31,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:31,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119011830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:31,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:31,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:31,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119011840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:31,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:31,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119011840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:31,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119011840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:31,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T02:22:31,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:31,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:31,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:31,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:31,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:31,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:32,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T02:22:32,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:32,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:32,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:32,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:32,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:32,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:32,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T02:22:32,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:32,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119012040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:32,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:32,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:32,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119012040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:32,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:32,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:32,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119012050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:32,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:32,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119012050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:32,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119012050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:32,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/41757ec5d1cf4951b6b9fb6d45231dbc 2024-11-09T02:22:32,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T02:22:32,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:32,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:32,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/e7fa25ace5c94befa875da9952532680 is 50, key is test_row_0/C:col10/1731118951702/Put/seqid=0 2024-11-09T02:22:32,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:32,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:32,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:32,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:32,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742100_1276 (size=12151) 2024-11-09T02:22:32,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/e7fa25ace5c94befa875da9952532680 2024-11-09T02:22:32,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/4d50155eb3d345d6b7160c3ede169655 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/4d50155eb3d345d6b7160c3ede169655 2024-11-09T02:22:32,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/4d50155eb3d345d6b7160c3ede169655, entries=150, sequenceid=196, filesize=11.9 K 2024-11-09T02:22:32,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/41757ec5d1cf4951b6b9fb6d45231dbc as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/41757ec5d1cf4951b6b9fb6d45231dbc 2024-11-09T02:22:32,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/41757ec5d1cf4951b6b9fb6d45231dbc, entries=150, sequenceid=196, filesize=11.9 K 2024-11-09T02:22:32,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/e7fa25ace5c94befa875da9952532680 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/e7fa25ace5c94befa875da9952532680 2024-11-09T02:22:32,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/e7fa25ace5c94befa875da9952532680, entries=150, sequenceid=196, filesize=11.9 K 2024-11-09T02:22:32,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for fbbc71fa787b8911356ef7975af46e80 in 485ms, sequenceid=196, compaction requested=true 2024-11-09T02:22:32,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:32,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbbc71fa787b8911356ef7975af46e80:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:22:32,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:32,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbbc71fa787b8911356ef7975af46e80:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:22:32,189 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:32,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:32,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbbc71fa787b8911356ef7975af46e80:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:22:32,189 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:32,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:32,190 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:32,190 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:32,190 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): fbbc71fa787b8911356ef7975af46e80/B is initiating minor compaction (all files) 2024-11-09T02:22:32,190 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): fbbc71fa787b8911356ef7975af46e80/A is initiating minor compaction (all files) 2024-11-09T02:22:32,190 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbbc71fa787b8911356ef7975af46e80/B in TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:32,190 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbbc71fa787b8911356ef7975af46e80/A in TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:32,190 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/cdca675112a24502ad3533328959efbf, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/26f5309c4a0e47469cd29bb924db9f96, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/41757ec5d1cf4951b6b9fb6d45231dbc] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp, totalSize=35.9 K 2024-11-09T02:22:32,190 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/f7ceb3108a6145ff9ac13e043bf5dd96, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/bc8ddf3498ad4b148d31470c4d67d016, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/4d50155eb3d345d6b7160c3ede169655] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp, totalSize=35.9 K 2024-11-09T02:22:32,190 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7ceb3108a6145ff9ac13e043bf5dd96, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731118949063 2024-11-09T02:22:32,190 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting cdca675112a24502ad3533328959efbf, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731118949063 2024-11-09T02:22:32,191 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc8ddf3498ad4b148d31470c4d67d016, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731118949402 2024-11-09T02:22:32,191 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 26f5309c4a0e47469cd29bb924db9f96, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731118949402 2024-11-09T02:22:32,192 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d50155eb3d345d6b7160c3ede169655, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731118951702 2024-11-09T02:22:32,192 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 41757ec5d1cf4951b6b9fb6d45231dbc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731118951702 2024-11-09T02:22:32,206 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbbc71fa787b8911356ef7975af46e80#A#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:32,207 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/df360f4d647947dbb341499ad3ab9fab is 50, key is test_row_0/A:col10/1731118951702/Put/seqid=0 2024-11-09T02:22:32,209 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbbc71fa787b8911356ef7975af46e80#B#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:32,210 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/6227dd0788f841428b761a7fbf61bcd7 is 50, key is test_row_0/B:col10/1731118951702/Put/seqid=0 2024-11-09T02:22:32,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742101_1277 (size=12595) 2024-11-09T02:22:32,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742102_1278 (size=12595) 2024-11-09T02:22:32,241 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/6227dd0788f841428b761a7fbf61bcd7 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/6227dd0788f841428b761a7fbf61bcd7 2024-11-09T02:22:32,264 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbbc71fa787b8911356ef7975af46e80/B of fbbc71fa787b8911356ef7975af46e80 into 6227dd0788f841428b761a7fbf61bcd7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:32,264 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:32,264 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., storeName=fbbc71fa787b8911356ef7975af46e80/B, priority=13, startTime=1731118952189; duration=0sec 2024-11-09T02:22:32,265 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:32,265 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbbc71fa787b8911356ef7975af46e80:B 2024-11-09T02:22:32,265 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:32,266 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:32,266 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): fbbc71fa787b8911356ef7975af46e80/C is initiating minor compaction (all files) 2024-11-09T02:22:32,266 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbbc71fa787b8911356ef7975af46e80/C in TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:32,266 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/2824c221be86420798fb4660847d06ba, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/e02215022b9d4ebda06e4fa371fac865, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/e7fa25ace5c94befa875da9952532680] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp, totalSize=35.9 K 2024-11-09T02:22:32,266 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 2824c221be86420798fb4660847d06ba, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731118949063 2024-11-09T02:22:32,267 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting e02215022b9d4ebda06e4fa371fac865, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731118949402 2024-11-09T02:22:32,267 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting e7fa25ace5c94befa875da9952532680, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731118951702 2024-11-09T02:22:32,276 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbbc71fa787b8911356ef7975af46e80#C#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:32,277 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/8b52f2650af74532b13ff33b79482bb8 is 50, key is test_row_0/C:col10/1731118951702/Put/seqid=0 2024-11-09T02:22:32,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742103_1279 (size=12595) 2024-11-09T02:22:32,296 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/8b52f2650af74532b13ff33b79482bb8 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/8b52f2650af74532b13ff33b79482bb8 2024-11-09T02:22:32,304 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbbc71fa787b8911356ef7975af46e80/C of fbbc71fa787b8911356ef7975af46e80 into 8b52f2650af74532b13ff33b79482bb8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:32,304 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:32,304 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., storeName=fbbc71fa787b8911356ef7975af46e80/C, priority=13, startTime=1731118952189; duration=0sec 2024-11-09T02:22:32,304 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:32,304 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbbc71fa787b8911356ef7975af46e80:C 2024-11-09T02:22:32,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T02:22:32,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:32,316 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing fbbc71fa787b8911356ef7975af46e80 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-09T02:22:32,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/4b2fcd88bd634cf8957d926809da8aca is 50, key is test_row_0/A:col10/1731118951723/Put/seqid=0 2024-11-09T02:22:32,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742104_1280 (size=12151) 2024-11-09T02:22:32,328 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/4b2fcd88bd634cf8957d926809da8aca 2024-11-09T02:22:32,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T02:22:32,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/7b32a76aa6e64b6caf5ca93955d97026 is 50, key is test_row_0/B:col10/1731118951723/Put/seqid=0 2024-11-09T02:22:32,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742105_1281 (size=12151) 2024-11-09T02:22:32,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:32,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:32,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:32,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119012371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:32,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:32,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:32,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119012384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:32,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:32,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:32,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119012403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:32,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:32,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:32,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119012403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:32,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:32,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:32,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119012403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:32,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:32,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:32,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119012510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:32,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:32,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:32,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119012510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:32,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:32,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:32,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119012510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:32,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:32,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:32,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119012519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:32,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:32,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:32,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119012519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:32,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:32,694 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/df360f4d647947dbb341499ad3ab9fab as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/df360f4d647947dbb341499ad3ab9fab 2024-11-09T02:22:32,700 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbbc71fa787b8911356ef7975af46e80/A of fbbc71fa787b8911356ef7975af46e80 into df360f4d647947dbb341499ad3ab9fab(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:32,700 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:32,700 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., storeName=fbbc71fa787b8911356ef7975af46e80/A, priority=13, startTime=1731118952189; duration=0sec 2024-11-09T02:22:32,700 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:32,700 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbbc71fa787b8911356ef7975af46e80:A 2024-11-09T02:22:32,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:32,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119012720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:32,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119012721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:32,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:32,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119012721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:32,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:32,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:32,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119012730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:32,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:32,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119012730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:32,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:32,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:32,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:32,740 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/7b32a76aa6e64b6caf5ca93955d97026 2024-11-09T02:22:32,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/193a53e2091a414a831a3e4a7a039e31 is 50, key is test_row_0/C:col10/1731118951723/Put/seqid=0 2024-11-09T02:22:32,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742106_1282 (size=12151) 2024-11-09T02:22:32,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T02:22:33,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119013030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:33,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119013030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:33,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119013030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:33,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:33,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119013040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:33,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119013040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:33,165 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/193a53e2091a414a831a3e4a7a039e31 2024-11-09T02:22:33,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/4b2fcd88bd634cf8957d926809da8aca as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/4b2fcd88bd634cf8957d926809da8aca 2024-11-09T02:22:33,174 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/4b2fcd88bd634cf8957d926809da8aca, entries=150, sequenceid=213, filesize=11.9 K 2024-11-09T02:22:33,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/7b32a76aa6e64b6caf5ca93955d97026 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/7b32a76aa6e64b6caf5ca93955d97026 2024-11-09T02:22:33,178 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/7b32a76aa6e64b6caf5ca93955d97026, entries=150, sequenceid=213, filesize=11.9 K 2024-11-09T02:22:33,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/193a53e2091a414a831a3e4a7a039e31 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/193a53e2091a414a831a3e4a7a039e31 2024-11-09T02:22:33,183 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/193a53e2091a414a831a3e4a7a039e31, entries=150, sequenceid=213, filesize=11.9 K 2024-11-09T02:22:33,184 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=134.18 KB/137400 for fbbc71fa787b8911356ef7975af46e80 in 868ms, sequenceid=213, compaction requested=false 2024-11-09T02:22:33,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:33,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:33,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-09T02:22:33,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-09T02:22:33,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-09T02:22:33,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4820 sec 2024-11-09T02:22:33,189 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.4870 sec 2024-11-09T02:22:33,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:33,543 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:22:33,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbbc71fa787b8911356ef7975af46e80 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-09T02:22:33,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/f2df994a989b45e5a87fddb2d2d968a6 is 50, key is test_row_0/A:col10/1731118952383/Put/seqid=0 2024-11-09T02:22:33,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742107_1283 (size=14541) 2024-11-09T02:22:33,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/f2df994a989b45e5a87fddb2d2d968a6 2024-11-09T02:22:33,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119013553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:33,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119013554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:33,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119013557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119013555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:33,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:33,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:33,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:33,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119013560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:33,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:33,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/0e9e94dae297488a81d901741f4ca5e0 is 50, key is test_row_0/B:col10/1731118952383/Put/seqid=0 2024-11-09T02:22:33,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742108_1284 (size=12151) 2024-11-09T02:22:33,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/0e9e94dae297488a81d901741f4ca5e0 2024-11-09T02:22:33,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/c86ae453c79a49829958862d0e241a93 is 50, key is test_row_0/C:col10/1731118952383/Put/seqid=0 2024-11-09T02:22:33,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742109_1285 (size=12151) 2024-11-09T02:22:33,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119013670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:33,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119013670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119013670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:33,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119013671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T02:22:33,850 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:22:33,851 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:22:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-09T02:22:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T02:22:33,852 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:22:33,853 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:22:33,853 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:22:33,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119013879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:33,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:33,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:33,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119013879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119013880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:33,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:33,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:33,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119013880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:33,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:33,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T02:22:33,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/c86ae453c79a49829958862d0e241a93 2024-11-09T02:22:33,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/f2df994a989b45e5a87fddb2d2d968a6 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/f2df994a989b45e5a87fddb2d2d968a6 2024-11-09T02:22:34,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/f2df994a989b45e5a87fddb2d2d968a6, entries=200, sequenceid=239, filesize=14.2 K 2024-11-09T02:22:34,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/0e9e94dae297488a81d901741f4ca5e0 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/0e9e94dae297488a81d901741f4ca5e0 2024-11-09T02:22:34,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-09T02:22:34,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:34,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:34,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:34,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:34,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:34,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:34,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/0e9e94dae297488a81d901741f4ca5e0, entries=150, sequenceid=239, filesize=11.9 K 2024-11-09T02:22:34,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/c86ae453c79a49829958862d0e241a93 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/c86ae453c79a49829958862d0e241a93 2024-11-09T02:22:34,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/c86ae453c79a49829958862d0e241a93, entries=150, sequenceid=239, filesize=11.9 K 2024-11-09T02:22:34,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for fbbc71fa787b8911356ef7975af46e80 in 475ms, sequenceid=239, compaction requested=true 2024-11-09T02:22:34,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:34,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbbc71fa787b8911356ef7975af46e80:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:22:34,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:34,018 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:34,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbbc71fa787b8911356ef7975af46e80:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:22:34,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:34,018 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:34,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbbc71fa787b8911356ef7975af46e80:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:22:34,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:34,019 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:34,019 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:34,019 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): fbbc71fa787b8911356ef7975af46e80/B is initiating minor compaction (all files) 2024-11-09T02:22:34,019 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): fbbc71fa787b8911356ef7975af46e80/A is initiating minor compaction (all files) 2024-11-09T02:22:34,019 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbbc71fa787b8911356ef7975af46e80/B in TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:34,019 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbbc71fa787b8911356ef7975af46e80/A in TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:34,019 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/6227dd0788f841428b761a7fbf61bcd7, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/7b32a76aa6e64b6caf5ca93955d97026, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/0e9e94dae297488a81d901741f4ca5e0] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp, totalSize=36.0 K 2024-11-09T02:22:34,019 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/df360f4d647947dbb341499ad3ab9fab, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/4b2fcd88bd634cf8957d926809da8aca, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/f2df994a989b45e5a87fddb2d2d968a6] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp, totalSize=38.4 K 2024-11-09T02:22:34,020 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting df360f4d647947dbb341499ad3ab9fab, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731118951702 2024-11-09T02:22:34,020 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 6227dd0788f841428b761a7fbf61bcd7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731118951702 2024-11-09T02:22:34,020 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b2fcd88bd634cf8957d926809da8aca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731118951717 2024-11-09T02:22:34,020 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b32a76aa6e64b6caf5ca93955d97026, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731118951717 2024-11-09T02:22:34,020 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2df994a989b45e5a87fddb2d2d968a6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731118952374 2024-11-09T02:22:34,020 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e9e94dae297488a81d901741f4ca5e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731118952374 2024-11-09T02:22:34,026 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbbc71fa787b8911356ef7975af46e80#B#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:34,027 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/111b2d673c594c069ec14ad20593efff is 50, key is test_row_0/B:col10/1731118952383/Put/seqid=0 2024-11-09T02:22:34,027 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbbc71fa787b8911356ef7975af46e80#A#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:34,028 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/831672cdd8644efeae81f96e353060f2 is 50, key is test_row_0/A:col10/1731118952383/Put/seqid=0 2024-11-09T02:22:34,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742110_1286 (size=12697) 2024-11-09T02:22:34,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742111_1287 (size=12697) 2024-11-09T02:22:34,052 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/831672cdd8644efeae81f96e353060f2 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/831672cdd8644efeae81f96e353060f2 2024-11-09T02:22:34,057 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbbc71fa787b8911356ef7975af46e80/A of fbbc71fa787b8911356ef7975af46e80 into 831672cdd8644efeae81f96e353060f2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:34,057 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:34,058 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., storeName=fbbc71fa787b8911356ef7975af46e80/A, priority=13, startTime=1731118954018; duration=0sec 2024-11-09T02:22:34,058 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:34,058 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbbc71fa787b8911356ef7975af46e80:A 2024-11-09T02:22:34,058 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:34,059 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:34,059 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): fbbc71fa787b8911356ef7975af46e80/C is initiating minor compaction (all files) 2024-11-09T02:22:34,059 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbbc71fa787b8911356ef7975af46e80/C in TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:34,059 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/8b52f2650af74532b13ff33b79482bb8, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/193a53e2091a414a831a3e4a7a039e31, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/c86ae453c79a49829958862d0e241a93] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp, totalSize=36.0 K 2024-11-09T02:22:34,059 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b52f2650af74532b13ff33b79482bb8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731118951702 2024-11-09T02:22:34,059 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 193a53e2091a414a831a3e4a7a039e31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731118951717 2024-11-09T02:22:34,072 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting c86ae453c79a49829958862d0e241a93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731118952374 2024-11-09T02:22:34,079 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbbc71fa787b8911356ef7975af46e80#C#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:34,080 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/2a2324aab8694905abf718d5e47f70e5 is 50, key is test_row_0/C:col10/1731118952383/Put/seqid=0 2024-11-09T02:22:34,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742112_1288 (size=12697) 2024-11-09T02:22:34,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-09T02:22:34,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:34,159 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing fbbc71fa787b8911356ef7975af46e80 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T02:22:34,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/747073693b40400fb2b4ccfbe0ad0a22 is 50, key is test_row_0/A:col10/1731118953554/Put/seqid=0 2024-11-09T02:22:34,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742113_1289 (size=12151) 2024-11-09T02:22:34,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T02:22:34,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:34,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:34,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:34,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119014217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:34,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:34,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119014218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:34,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:34,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:34,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:34,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119014219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:34,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:34,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:34,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119014220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:34,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:34,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:34,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119014329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:34,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:34,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:34,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119014330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:34,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:34,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:34,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119014330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:34,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:34,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:34,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119014330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:34,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:34,444 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/111b2d673c594c069ec14ad20593efff as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/111b2d673c594c069ec14ad20593efff 2024-11-09T02:22:34,448 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbbc71fa787b8911356ef7975af46e80/B of fbbc71fa787b8911356ef7975af46e80 into 111b2d673c594c069ec14ad20593efff(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:34,448 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:34,448 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., storeName=fbbc71fa787b8911356ef7975af46e80/B, priority=13, startTime=1731118954018; duration=0sec 2024-11-09T02:22:34,448 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:34,448 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbbc71fa787b8911356ef7975af46e80:B 2024-11-09T02:22:34,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T02:22:34,488 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/2a2324aab8694905abf718d5e47f70e5 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/2a2324aab8694905abf718d5e47f70e5 2024-11-09T02:22:34,493 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbbc71fa787b8911356ef7975af46e80/C of fbbc71fa787b8911356ef7975af46e80 into 2a2324aab8694905abf718d5e47f70e5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:34,493 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:34,493 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., storeName=fbbc71fa787b8911356ef7975af46e80/C, priority=13, startTime=1731118954018; duration=0sec 2024-11-09T02:22:34,493 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:34,493 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbbc71fa787b8911356ef7975af46e80:C 2024-11-09T02:22:34,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:34,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119014540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:34,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:34,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:34,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119014540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:34,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:34,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119014540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:34,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:34,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:34,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119014540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:34,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:34,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:34,567 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/747073693b40400fb2b4ccfbe0ad0a22 2024-11-09T02:22:34,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:34,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119014570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:34,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/9166b799c69b4f638a434aef868bf0e6 is 50, key is test_row_0/B:col10/1731118953554/Put/seqid=0 2024-11-09T02:22:34,574 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_1' in fbbc71fa787b8911356ef7975af46e80 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:22:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:34,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742114_1290 (size=12151) 2024-11-09T02:22:34,578 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/9166b799c69b4f638a434aef868bf0e6 2024-11-09T02:22:34,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/8e987d1d382044f6aaa263c44e12db10 is 50, key is test_row_0/C:col10/1731118953554/Put/seqid=0 2024-11-09T02:22:34,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742115_1291 (size=12151) 2024-11-09T02:22:34,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119014849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:34,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:34,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119014849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:34,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:34,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:34,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119014850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:34,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:34,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119014850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:34,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:34,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:34,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T02:22:35,001 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/8e987d1d382044f6aaa263c44e12db10 2024-11-09T02:22:35,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/747073693b40400fb2b4ccfbe0ad0a22 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/747073693b40400fb2b4ccfbe0ad0a22 2024-11-09T02:22:35,009 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/747073693b40400fb2b4ccfbe0ad0a22, entries=150, sequenceid=251, filesize=11.9 K 2024-11-09T02:22:35,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/9166b799c69b4f638a434aef868bf0e6 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/9166b799c69b4f638a434aef868bf0e6 2024-11-09T02:22:35,015 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/9166b799c69b4f638a434aef868bf0e6, entries=150, sequenceid=251, filesize=11.9 K 2024-11-09T02:22:35,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/8e987d1d382044f6aaa263c44e12db10 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/8e987d1d382044f6aaa263c44e12db10 2024-11-09T02:22:35,020 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/8e987d1d382044f6aaa263c44e12db10, entries=150, sequenceid=251, filesize=11.9 K 2024-11-09T02:22:35,021 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for fbbc71fa787b8911356ef7975af46e80 in 862ms, sequenceid=251, compaction requested=false 2024-11-09T02:22:35,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:35,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:35,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-09T02:22:35,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-09T02:22:35,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-09T02:22:35,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1700 sec 2024-11-09T02:22:35,026 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.1740 sec 2024-11-09T02:22:35,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:35,361 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:22:35,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbbc71fa787b8911356ef7975af46e80 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-09T02:22:35,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/21996ebf1cb34c1f910e03344cd0521b is 50, key is test_row_0/A:col10/1731118954219/Put/seqid=0 2024-11-09T02:22:35,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:35,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119015367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:35,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119015369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:35,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119015369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:35,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:35,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:35,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:35,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119015369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:35,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:35,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:35,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:35,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742116_1292 (size=12301) 2024-11-09T02:22:35,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/21996ebf1cb34c1f910e03344cd0521b 2024-11-09T02:22:35,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/dde13a5c48924c0c9f0b814e5c60ccc1 is 50, key is test_row_0/B:col10/1731118954219/Put/seqid=0 2024-11-09T02:22:35,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742117_1293 (size=12301) 2024-11-09T02:22:35,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:35,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119015480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:35,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:35,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119015480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:35,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:35,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:35,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:35,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:35,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:35,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:35,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:35,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119015480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:35,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:35,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:35,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:35,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:35,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119015480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:35,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:35,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:35,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:35,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:35,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119015690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119015690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:35,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119015690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:35,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119015690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:35,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:35,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:35,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/dde13a5c48924c0c9f0b814e5c60ccc1 2024-11-09T02:22:35,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/71bfc22acc864caa9408485aed212ad1 is 50, key is test_row_0/C:col10/1731118954219/Put/seqid=0 2024-11-09T02:22:35,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742118_1294 (size=12301) 2024-11-09T02:22:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T02:22:36,000 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:22:36,000 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:22:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-09T02:22:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T02:22:36,002 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:22:36,002 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:22:36,003 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:22:36,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119016000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:36,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:36,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119016000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:36,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119016000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119016000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T02:22:36,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-09T02:22:36,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:36,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:36,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:36,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:36,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:36,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:36,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/71bfc22acc864caa9408485aed212ad1 2024-11-09T02:22:36,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/21996ebf1cb34c1f910e03344cd0521b as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/21996ebf1cb34c1f910e03344cd0521b 2024-11-09T02:22:36,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/21996ebf1cb34c1f910e03344cd0521b, entries=150, sequenceid=279, filesize=12.0 K 2024-11-09T02:22:36,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/dde13a5c48924c0c9f0b814e5c60ccc1 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/dde13a5c48924c0c9f0b814e5c60ccc1 2024-11-09T02:22:36,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/dde13a5c48924c0c9f0b814e5c60ccc1, entries=150, sequenceid=279, filesize=12.0 K 2024-11-09T02:22:36,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/71bfc22acc864caa9408485aed212ad1 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/71bfc22acc864caa9408485aed212ad1 2024-11-09T02:22:36,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/71bfc22acc864caa9408485aed212ad1, entries=150, sequenceid=279, filesize=12.0 K 2024-11-09T02:22:36,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for fbbc71fa787b8911356ef7975af46e80 in 860ms, sequenceid=279, compaction requested=true 2024-11-09T02:22:36,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:36,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbbc71fa787b8911356ef7975af46e80:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:22:36,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:36,222 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:36,222 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:36,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbbc71fa787b8911356ef7975af46e80:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:22:36,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:36,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbbc71fa787b8911356ef7975af46e80:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:22:36,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:36,222 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:36,222 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:36,223 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): fbbc71fa787b8911356ef7975af46e80/B is initiating minor compaction (all files) 2024-11-09T02:22:36,223 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): fbbc71fa787b8911356ef7975af46e80/A is initiating minor compaction (all files) 2024-11-09T02:22:36,223 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbbc71fa787b8911356ef7975af46e80/B in TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:36,223 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbbc71fa787b8911356ef7975af46e80/A in TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:36,223 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/831672cdd8644efeae81f96e353060f2, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/747073693b40400fb2b4ccfbe0ad0a22, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/21996ebf1cb34c1f910e03344cd0521b] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp, totalSize=36.3 K 2024-11-09T02:22:36,223 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/111b2d673c594c069ec14ad20593efff, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/9166b799c69b4f638a434aef868bf0e6, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/dde13a5c48924c0c9f0b814e5c60ccc1] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp, totalSize=36.3 K 2024-11-09T02:22:36,223 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 831672cdd8644efeae81f96e353060f2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731118952374 2024-11-09T02:22:36,223 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 111b2d673c594c069ec14ad20593efff, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731118952374 2024-11-09T02:22:36,223 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 9166b799c69b4f638a434aef868bf0e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731118953551 2024-11-09T02:22:36,223 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 747073693b40400fb2b4ccfbe0ad0a22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731118953551 2024-11-09T02:22:36,223 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting dde13a5c48924c0c9f0b814e5c60ccc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731118954218 2024-11-09T02:22:36,223 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21996ebf1cb34c1f910e03344cd0521b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731118954218 2024-11-09T02:22:36,230 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbbc71fa787b8911356ef7975af46e80#B#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:36,230 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbbc71fa787b8911356ef7975af46e80#A#compaction#250 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:36,230 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/1d084ab4a12f4aac87c8e3fa803076ab is 50, key is test_row_0/B:col10/1731118954219/Put/seqid=0 2024-11-09T02:22:36,230 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/140bc6d8c1654e6e8580d112c71c6c69 is 50, key is test_row_0/A:col10/1731118954219/Put/seqid=0 2024-11-09T02:22:36,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742119_1295 (size=12949) 2024-11-09T02:22:36,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742120_1296 (size=12949) 2024-11-09T02:22:36,250 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/1d084ab4a12f4aac87c8e3fa803076ab as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/1d084ab4a12f4aac87c8e3fa803076ab 2024-11-09T02:22:36,250 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/140bc6d8c1654e6e8580d112c71c6c69 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/140bc6d8c1654e6e8580d112c71c6c69 2024-11-09T02:22:36,256 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbbc71fa787b8911356ef7975af46e80/B of fbbc71fa787b8911356ef7975af46e80 into 1d084ab4a12f4aac87c8e3fa803076ab(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:36,256 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbbc71fa787b8911356ef7975af46e80/A of fbbc71fa787b8911356ef7975af46e80 into 140bc6d8c1654e6e8580d112c71c6c69(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:36,256 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:36,256 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:36,256 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., storeName=fbbc71fa787b8911356ef7975af46e80/A, priority=13, startTime=1731118956221; duration=0sec 2024-11-09T02:22:36,256 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., storeName=fbbc71fa787b8911356ef7975af46e80/B, priority=13, startTime=1731118956222; duration=0sec 2024-11-09T02:22:36,256 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:36,256 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:36,256 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbbc71fa787b8911356ef7975af46e80:B 2024-11-09T02:22:36,256 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbbc71fa787b8911356ef7975af46e80:A 2024-11-09T02:22:36,256 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:36,257 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:36,257 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): fbbc71fa787b8911356ef7975af46e80/C is initiating minor compaction (all files) 2024-11-09T02:22:36,257 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbbc71fa787b8911356ef7975af46e80/C in TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:36,257 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/2a2324aab8694905abf718d5e47f70e5, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/8e987d1d382044f6aaa263c44e12db10, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/71bfc22acc864caa9408485aed212ad1] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp, totalSize=36.3 K 2024-11-09T02:22:36,257 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a2324aab8694905abf718d5e47f70e5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731118952374 2024-11-09T02:22:36,258 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e987d1d382044f6aaa263c44e12db10, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731118953551 2024-11-09T02:22:36,258 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 71bfc22acc864caa9408485aed212ad1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731118954218 2024-11-09T02:22:36,264 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbbc71fa787b8911356ef7975af46e80#C#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:36,265 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/b671cd1f7a5d40ab885ba705481ccdfe is 50, key is test_row_0/C:col10/1731118954219/Put/seqid=0 2024-11-09T02:22:36,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742121_1297 (size=12949) 2024-11-09T02:22:36,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-09T02:22:36,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:36,308 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing fbbc71fa787b8911356ef7975af46e80 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-09T02:22:36,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/c7d2c0b470774d0199aca5b70353c0b2 is 50, key is test_row_0/A:col10/1731118955368/Put/seqid=0 2024-11-09T02:22:36,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T02:22:36,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742122_1298 (size=12301) 2024-11-09T02:22:36,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:36,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:36,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119016535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:36,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:36,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:36,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119016535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:36,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119016539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:36,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119016539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:36,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119016590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:36,592 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_1' in fbbc71fa787b8911356ef7975af46e80 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:22:36,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T02:22:36,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119016649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:36,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119016650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:36,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119016650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:36,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119016650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:36,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:36,680 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/b671cd1f7a5d40ab885ba705481ccdfe as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/b671cd1f7a5d40ab885ba705481ccdfe 2024-11-09T02:22:36,684 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbbc71fa787b8911356ef7975af46e80/C of fbbc71fa787b8911356ef7975af46e80 into b671cd1f7a5d40ab885ba705481ccdfe(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:36,684 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:36,684 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., storeName=fbbc71fa787b8911356ef7975af46e80/C, priority=13, startTime=1731118956222; duration=0sec 2024-11-09T02:22:36,684 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:36,685 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbbc71fa787b8911356ef7975af46e80:C 2024-11-09T02:22:36,726 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/c7d2c0b470774d0199aca5b70353c0b2 2024-11-09T02:22:36,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/37d1d4cca7de4dfda17ba8bcbd3f987b is 50, key is test_row_0/B:col10/1731118955368/Put/seqid=0 2024-11-09T02:22:36,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742123_1299 (size=12301) 2024-11-09T02:22:36,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:36,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119016860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:36,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:36,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119016860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:36,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:36,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:36,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:36,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119016860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:36,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:36,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:36,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119016860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:36,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:36,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:37,136 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/37d1d4cca7de4dfda17ba8bcbd3f987b 2024-11-09T02:22:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T02:22:37,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/8e076111a75a4f0b8668014efa69d051 is 50, key is test_row_0/C:col10/1731118955368/Put/seqid=0 2024-11-09T02:22:37,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742124_1300 (size=12301) 2024-11-09T02:22:37,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119017170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:37,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:37,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:37,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:37,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119017170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:37,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119017170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:37,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:37,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:37,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:37,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:37,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:37,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:37,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119017170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:37,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:37,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:37,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:37,545 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/8e076111a75a4f0b8668014efa69d051 2024-11-09T02:22:37,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/c7d2c0b470774d0199aca5b70353c0b2 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/c7d2c0b470774d0199aca5b70353c0b2 2024-11-09T02:22:37,554 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/c7d2c0b470774d0199aca5b70353c0b2, entries=150, sequenceid=291, filesize=12.0 K 2024-11-09T02:22:37,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/37d1d4cca7de4dfda17ba8bcbd3f987b as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/37d1d4cca7de4dfda17ba8bcbd3f987b 2024-11-09T02:22:37,558 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/37d1d4cca7de4dfda17ba8bcbd3f987b, entries=150, sequenceid=291, filesize=12.0 K 2024-11-09T02:22:37,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/8e076111a75a4f0b8668014efa69d051 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/8e076111a75a4f0b8668014efa69d051 2024-11-09T02:22:37,563 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/8e076111a75a4f0b8668014efa69d051, entries=150, sequenceid=291, filesize=12.0 K 2024-11-09T02:22:37,564 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for fbbc71fa787b8911356ef7975af46e80 in 1256ms, sequenceid=291, compaction requested=false 2024-11-09T02:22:37,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:37,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:37,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-09T02:22:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-09T02:22:37,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-09T02:22:37,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5630 sec 2024-11-09T02:22:37,569 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.5680 sec 2024-11-09T02:22:37,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:37,683 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:22:37,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbbc71fa787b8911356ef7975af46e80 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-09T02:22:37,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119017685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:37,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:37,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:37,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:37,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:37,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119017687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:37,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:37,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:37,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119017688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:37,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:37,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:37,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:37,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:37,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:37,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:37,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119017689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:37,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:37,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:37,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:37,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/1a9c8dad268a43889300ad33cfa68d47 is 50, key is test_row_0/A:col10/1731118957682/Put/seqid=0 2024-11-09T02:22:37,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742125_1301 (size=14741) 2024-11-09T02:22:37,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:37,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119017799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:37,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:37,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:37,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:37,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119017800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:37,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:37,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:37,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:37,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:37,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:37,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119017800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:37,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:37,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:37,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:37,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:37,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119017800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:37,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:37,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:37,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:38,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119018009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:38,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:38,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:38,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119018010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:38,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119018010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:38,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:38,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119018010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:38,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/1a9c8dad268a43889300ad33cfa68d47 2024-11-09T02:22:38,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/6e0218f28cce49babeae5e012e371e9b is 50, key is test_row_0/B:col10/1731118957682/Put/seqid=0 2024-11-09T02:22:38,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742126_1302 (size=12301) 2024-11-09T02:22:38,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T02:22:38,150 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:22:38,151 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:22:38,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-09T02:22:38,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T02:22:38,152 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:22:38,153 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:22:38,153 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:22:38,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T02:22:38,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-09T02:22:38,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:38,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:38,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:38,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:38,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:38,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:38,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119018319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:38,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:38,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:38,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119018320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:38,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119018320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:38,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:38,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119018320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:38,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-09T02:22:38,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:38,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:38,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:38,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:38,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:38,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:38,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T02:22:38,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/6e0218f28cce49babeae5e012e371e9b 2024-11-09T02:22:38,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/746e634d67da40c6bcc8fb8eeca8da51 is 50, key is test_row_0/C:col10/1731118957682/Put/seqid=0 2024-11-09T02:22:38,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742127_1303 (size=12301) 2024-11-09T02:22:38,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-09T02:22:38,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:38,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:38,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:38,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:38,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:38,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-09T02:22:38,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:38,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:38,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:38,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:38,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T02:22:38,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119018829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:38,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:38,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:38,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:38,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119018829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:38,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:38,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:38,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:38,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119018830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:38,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:38,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119018830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:38,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-09T02:22:38,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:38,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:38,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:38,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:38,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:38,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:38,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/746e634d67da40c6bcc8fb8eeca8da51 2024-11-09T02:22:38,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/1a9c8dad268a43889300ad33cfa68d47 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/1a9c8dad268a43889300ad33cfa68d47 2024-11-09T02:22:38,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/1a9c8dad268a43889300ad33cfa68d47, entries=200, sequenceid=321, filesize=14.4 K 2024-11-09T02:22:38,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/6e0218f28cce49babeae5e012e371e9b as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/6e0218f28cce49babeae5e012e371e9b 2024-11-09T02:22:38,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/6e0218f28cce49babeae5e012e371e9b, entries=150, sequenceid=321, filesize=12.0 K 2024-11-09T02:22:38,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/746e634d67da40c6bcc8fb8eeca8da51 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/746e634d67da40c6bcc8fb8eeca8da51 2024-11-09T02:22:38,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/746e634d67da40c6bcc8fb8eeca8da51, entries=150, sequenceid=321, filesize=12.0 K 2024-11-09T02:22:38,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for fbbc71fa787b8911356ef7975af46e80 in 1294ms, sequenceid=321, compaction requested=true 2024-11-09T02:22:38,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:38,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbbc71fa787b8911356ef7975af46e80:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:22:38,977 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:38,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:38,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbbc71fa787b8911356ef7975af46e80:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:22:38,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:38,977 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:38,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbbc71fa787b8911356ef7975af46e80:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:22:38,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:38,978 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:38,978 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): fbbc71fa787b8911356ef7975af46e80/A is initiating minor compaction (all files) 2024-11-09T02:22:38,978 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:38,978 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): fbbc71fa787b8911356ef7975af46e80/B is initiating minor compaction (all files) 2024-11-09T02:22:38,978 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbbc71fa787b8911356ef7975af46e80/A in TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:38,978 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbbc71fa787b8911356ef7975af46e80/B in TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:38,978 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/140bc6d8c1654e6e8580d112c71c6c69, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/c7d2c0b470774d0199aca5b70353c0b2, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/1a9c8dad268a43889300ad33cfa68d47] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp, totalSize=39.1 K 2024-11-09T02:22:38,978 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/1d084ab4a12f4aac87c8e3fa803076ab, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/37d1d4cca7de4dfda17ba8bcbd3f987b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/6e0218f28cce49babeae5e012e371e9b] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp, totalSize=36.7 K 2024-11-09T02:22:38,979 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 140bc6d8c1654e6e8580d112c71c6c69, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731118954218 2024-11-09T02:22:38,979 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d084ab4a12f4aac87c8e3fa803076ab, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731118954218 2024-11-09T02:22:38,979 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7d2c0b470774d0199aca5b70353c0b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731118955365 2024-11-09T02:22:38,979 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 37d1d4cca7de4dfda17ba8bcbd3f987b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731118955365 2024-11-09T02:22:38,979 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e0218f28cce49babeae5e012e371e9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731118957681 2024-11-09T02:22:38,979 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a9c8dad268a43889300ad33cfa68d47, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731118956533 2024-11-09T02:22:38,986 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbbc71fa787b8911356ef7975af46e80#A#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:38,986 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/dbf1503a001c401b8646afb02e00a862 is 50, key is test_row_0/A:col10/1731118957682/Put/seqid=0 2024-11-09T02:22:38,988 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbbc71fa787b8911356ef7975af46e80#B#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:38,988 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/5fb8514177e94567b9bb5a795b8d46c4 is 50, key is test_row_0/B:col10/1731118957682/Put/seqid=0 2024-11-09T02:22:38,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742128_1304 (size=13051) 2024-11-09T02:22:38,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742129_1305 (size=13051) 2024-11-09T02:22:39,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-09T02:22:39,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:39,069 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing fbbc71fa787b8911356ef7975af46e80 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-09T02:22:39,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/9ec707a19e9c4e78855f2a4af07d3a94 is 50, key is test_row_0/A:col10/1731118957688/Put/seqid=0 2024-11-09T02:22:39,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742130_1306 (size=9857) 2024-11-09T02:22:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T02:22:39,395 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/dbf1503a001c401b8646afb02e00a862 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/dbf1503a001c401b8646afb02e00a862 2024-11-09T02:22:39,398 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/5fb8514177e94567b9bb5a795b8d46c4 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/5fb8514177e94567b9bb5a795b8d46c4 2024-11-09T02:22:39,401 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbbc71fa787b8911356ef7975af46e80/A of fbbc71fa787b8911356ef7975af46e80 into dbf1503a001c401b8646afb02e00a862(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:39,401 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:39,401 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., storeName=fbbc71fa787b8911356ef7975af46e80/A, priority=13, startTime=1731118958977; duration=0sec 2024-11-09T02:22:39,401 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:39,401 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbbc71fa787b8911356ef7975af46e80:A 2024-11-09T02:22:39,402 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:39,402 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:39,402 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): fbbc71fa787b8911356ef7975af46e80/C is initiating minor compaction (all files) 2024-11-09T02:22:39,403 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbbc71fa787b8911356ef7975af46e80/C in TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:39,403 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/b671cd1f7a5d40ab885ba705481ccdfe, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/8e076111a75a4f0b8668014efa69d051, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/746e634d67da40c6bcc8fb8eeca8da51] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp, totalSize=36.7 K 2024-11-09T02:22:39,403 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting b671cd1f7a5d40ab885ba705481ccdfe, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731118954218 2024-11-09T02:22:39,403 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e076111a75a4f0b8668014efa69d051, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731118955365 2024-11-09T02:22:39,404 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 746e634d67da40c6bcc8fb8eeca8da51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731118957681 2024-11-09T02:22:39,404 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbbc71fa787b8911356ef7975af46e80/B of fbbc71fa787b8911356ef7975af46e80 into 5fb8514177e94567b9bb5a795b8d46c4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:39,404 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:39,404 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., storeName=fbbc71fa787b8911356ef7975af46e80/B, priority=13, startTime=1731118958977; duration=0sec 2024-11-09T02:22:39,404 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:39,404 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbbc71fa787b8911356ef7975af46e80:B 2024-11-09T02:22:39,410 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbbc71fa787b8911356ef7975af46e80#C#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:39,411 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/3349acf14fc7429898c4c8c4656580dd is 50, key is test_row_0/C:col10/1731118957682/Put/seqid=0 2024-11-09T02:22:39,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742131_1307 (size=13051) 2024-11-09T02:22:39,476 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/9ec707a19e9c4e78855f2a4af07d3a94 2024-11-09T02:22:39,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/de053b937b25483aa3f6831ae6a12917 is 50, key is test_row_0/B:col10/1731118957688/Put/seqid=0 2024-11-09T02:22:39,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742132_1308 (size=9857) 2024-11-09T02:22:39,823 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/3349acf14fc7429898c4c8c4656580dd as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/3349acf14fc7429898c4c8c4656580dd 2024-11-09T02:22:39,827 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbbc71fa787b8911356ef7975af46e80/C of fbbc71fa787b8911356ef7975af46e80 into 3349acf14fc7429898c4c8c4656580dd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:39,827 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:39,827 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., storeName=fbbc71fa787b8911356ef7975af46e80/C, priority=13, startTime=1731118958977; duration=0sec 2024-11-09T02:22:39,827 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:39,827 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbbc71fa787b8911356ef7975af46e80:C 2024-11-09T02:22:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:39,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. as already flushing 2024-11-09T02:22:39,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:39,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119019866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119019866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:39,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:39,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:39,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119019867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:39,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:39,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119019867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:39,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:39,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:39,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:39,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:39,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:39,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:39,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:39,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:39,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:39,887 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/de053b937b25483aa3f6831ae6a12917 2024-11-09T02:22:39,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/1dd772b876db4045b518a416248904b8 is 50, key is test_row_0/C:col10/1731118957688/Put/seqid=0 2024-11-09T02:22:39,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742133_1309 (size=9857) 2024-11-09T02:22:39,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119019979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:39,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:39,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119019980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:39,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:39,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:39,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:39,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:39,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:39,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119019980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:39,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:39,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:39,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:39,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119019980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:39,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:39,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:39,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:40,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:40,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119020190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:40,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:40,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119020190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:40,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:40,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119020190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:40,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:40,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119020190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:40,297 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/1dd772b876db4045b518a416248904b8 2024-11-09T02:22:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T02:22:40,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/9ec707a19e9c4e78855f2a4af07d3a94 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/9ec707a19e9c4e78855f2a4af07d3a94 2024-11-09T02:22:40,305 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/9ec707a19e9c4e78855f2a4af07d3a94, entries=100, sequenceid=328, filesize=9.6 K 2024-11-09T02:22:40,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/de053b937b25483aa3f6831ae6a12917 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/de053b937b25483aa3f6831ae6a12917 2024-11-09T02:22:40,309 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/de053b937b25483aa3f6831ae6a12917, entries=100, sequenceid=328, filesize=9.6 K 2024-11-09T02:22:40,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/1dd772b876db4045b518a416248904b8 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/1dd772b876db4045b518a416248904b8 2024-11-09T02:22:40,314 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/1dd772b876db4045b518a416248904b8, entries=100, sequenceid=328, filesize=9.6 K 2024-11-09T02:22:40,315 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for fbbc71fa787b8911356ef7975af46e80 in 1245ms, sequenceid=328, compaction requested=false 2024-11-09T02:22:40,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:40,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:40,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-09T02:22:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-09T02:22:40,318 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-09T02:22:40,318 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1630 sec 2024-11-09T02:22:40,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.1680 sec 2024-11-09T02:22:40,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:40,535 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:22:40,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbbc71fa787b8911356ef7975af46e80 3/3 column families, dataSize=194.56 KB heapSize=510.52 KB 2024-11-09T02:22:40,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:40,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119020536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:40,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:40,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:40,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119020536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:40,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:40,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119020536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:40,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119020539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:40,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/f0db8c3a13fd458eb4800e91e4d81017 is 50, key is test_row_0/A:col10/1731118960501/Put/seqid=0 2024-11-09T02:22:40,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742134_1310 (size=12301) 2024-11-09T02:22:40,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:40,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34366 deadline: 1731119020609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:40,611 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_1' in fbbc71fa787b8911356ef7975af46e80 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:22:40,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:40,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:40,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119020650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:40,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:40,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:40,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119020650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:40,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:40,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119020650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:40,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119020650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:40,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:40,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119020860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:40,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119020860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:40,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119020860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:40,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119020860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:40,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/f0db8c3a13fd458eb4800e91e4d81017 2024-11-09T02:22:40,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/9fbc5437e0f0413b9cb5a0a9bbaab1d1 is 50, key is test_row_0/B:col10/1731118960501/Put/seqid=0 2024-11-09T02:22:40,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742135_1311 (size=12301) 2024-11-09T02:22:41,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:41,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119021169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:41,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:41,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119021169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:41,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:41,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119021169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:41,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:41,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119021170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:41,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/9fbc5437e0f0413b9cb5a0a9bbaab1d1 2024-11-09T02:22:41,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/c879deb3c7e14898bd1c65a35a253b42 is 50, key is test_row_0/C:col10/1731118960501/Put/seqid=0 2024-11-09T02:22:41,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742136_1312 (size=12301) 2024-11-09T02:22:41,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34376 deadline: 1731119021680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:41,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34420 deadline: 1731119021680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:41,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34430 deadline: 1731119021680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:41,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34436 deadline: 1731119021680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:41,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:41,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:41,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:22:41,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/c879deb3c7e14898bd1c65a35a253b42 2024-11-09T02:22:41,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/f0db8c3a13fd458eb4800e91e4d81017 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/f0db8c3a13fd458eb4800e91e4d81017 2024-11-09T02:22:41,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/f0db8c3a13fd458eb4800e91e4d81017, entries=150, sequenceid=363, filesize=12.0 K 2024-11-09T02:22:41,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/9fbc5437e0f0413b9cb5a0a9bbaab1d1 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/9fbc5437e0f0413b9cb5a0a9bbaab1d1 2024-11-09T02:22:41,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/9fbc5437e0f0413b9cb5a0a9bbaab1d1, entries=150, sequenceid=363, filesize=12.0 K 2024-11-09T02:22:41,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/c879deb3c7e14898bd1c65a35a253b42 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/c879deb3c7e14898bd1c65a35a253b42 2024-11-09T02:22:41,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/c879deb3c7e14898bd1c65a35a253b42, entries=150, sequenceid=363, filesize=12.0 K 2024-11-09T02:22:41,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=6.71 KB/6870 for fbbc71fa787b8911356ef7975af46e80 in 1257ms, sequenceid=363, compaction requested=true 2024-11-09T02:22:41,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:41,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbbc71fa787b8911356ef7975af46e80:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:22:41,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:41,792 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:41,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbbc71fa787b8911356ef7975af46e80:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:22:41,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:41,792 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:41,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbbc71fa787b8911356ef7975af46e80:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:22:41,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:41,793 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:41,793 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:41,793 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): fbbc71fa787b8911356ef7975af46e80/A is initiating minor compaction (all files) 2024-11-09T02:22:41,793 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): fbbc71fa787b8911356ef7975af46e80/B is initiating minor compaction (all files) 2024-11-09T02:22:41,793 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbbc71fa787b8911356ef7975af46e80/A in TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:41,793 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbbc71fa787b8911356ef7975af46e80/B in TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:41,793 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/5fb8514177e94567b9bb5a795b8d46c4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/de053b937b25483aa3f6831ae6a12917, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/9fbc5437e0f0413b9cb5a0a9bbaab1d1] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp, totalSize=34.4 K 2024-11-09T02:22:41,793 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/dbf1503a001c401b8646afb02e00a862, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/9ec707a19e9c4e78855f2a4af07d3a94, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/f0db8c3a13fd458eb4800e91e4d81017] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp, totalSize=34.4 K 2024-11-09T02:22:41,794 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbf1503a001c401b8646afb02e00a862, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731118957681 2024-11-09T02:22:41,794 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fb8514177e94567b9bb5a795b8d46c4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731118957681 2024-11-09T02:22:41,794 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ec707a19e9c4e78855f2a4af07d3a94, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731118957687 2024-11-09T02:22:41,794 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting de053b937b25483aa3f6831ae6a12917, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731118957687 2024-11-09T02:22:41,795 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0db8c3a13fd458eb4800e91e4d81017, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731118959866 2024-11-09T02:22:41,795 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fbc5437e0f0413b9cb5a0a9bbaab1d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731118959866 2024-11-09T02:22:41,801 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbbc71fa787b8911356ef7975af46e80#B#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:41,802 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/422bbc9b8d8d42df8ffef53d85bac646 is 50, key is test_row_0/B:col10/1731118960501/Put/seqid=0 2024-11-09T02:22:41,803 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbbc71fa787b8911356ef7975af46e80#A#compaction#268 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:41,803 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/dc7b7edbc24946f5a221bdb36c00bca8 is 50, key is test_row_0/A:col10/1731118960501/Put/seqid=0 2024-11-09T02:22:41,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742137_1313 (size=13153) 2024-11-09T02:22:41,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742138_1314 (size=13153) 2024-11-09T02:22:41,818 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/dc7b7edbc24946f5a221bdb36c00bca8 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/dc7b7edbc24946f5a221bdb36c00bca8 2024-11-09T02:22:41,819 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/422bbc9b8d8d42df8ffef53d85bac646 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/422bbc9b8d8d42df8ffef53d85bac646 2024-11-09T02:22:41,825 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbbc71fa787b8911356ef7975af46e80/A of fbbc71fa787b8911356ef7975af46e80 into dc7b7edbc24946f5a221bdb36c00bca8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:41,826 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbbc71fa787b8911356ef7975af46e80/B of fbbc71fa787b8911356ef7975af46e80 into 422bbc9b8d8d42df8ffef53d85bac646(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:41,826 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:41,826 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:41,826 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., storeName=fbbc71fa787b8911356ef7975af46e80/B, priority=13, startTime=1731118961792; duration=0sec 2024-11-09T02:22:41,826 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., storeName=fbbc71fa787b8911356ef7975af46e80/A, priority=13, startTime=1731118961792; duration=0sec 2024-11-09T02:22:41,826 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:22:41,826 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbbc71fa787b8911356ef7975af46e80:B 2024-11-09T02:22:41,826 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:41,826 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbbc71fa787b8911356ef7975af46e80:A 2024-11-09T02:22:41,826 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:22:41,827 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:22:41,827 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): fbbc71fa787b8911356ef7975af46e80/C is initiating minor compaction (all files) 2024-11-09T02:22:41,827 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbbc71fa787b8911356ef7975af46e80/C in TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:41,828 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/3349acf14fc7429898c4c8c4656580dd, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/1dd772b876db4045b518a416248904b8, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/c879deb3c7e14898bd1c65a35a253b42] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp, totalSize=34.4 K 2024-11-09T02:22:41,828 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 3349acf14fc7429898c4c8c4656580dd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731118957681 2024-11-09T02:22:41,828 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dd772b876db4045b518a416248904b8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731118957687 2024-11-09T02:22:41,828 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting c879deb3c7e14898bd1c65a35a253b42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731118959866 2024-11-09T02:22:41,837 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbbc71fa787b8911356ef7975af46e80#C#compaction#269 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:22:41,837 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/c0d634a1d96d4d07a9ee2c80280a4e02 is 50, key is test_row_0/C:col10/1731118960501/Put/seqid=0 2024-11-09T02:22:41,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742139_1315 (size=13153) 2024-11-09T02:22:41,850 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/c0d634a1d96d4d07a9ee2c80280a4e02 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/c0d634a1d96d4d07a9ee2c80280a4e02 2024-11-09T02:22:41,856 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbbc71fa787b8911356ef7975af46e80/C of fbbc71fa787b8911356ef7975af46e80 into c0d634a1d96d4d07a9ee2c80280a4e02(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:22:41,856 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbbc71fa787b8911356ef7975af46e80: 2024-11-09T02:22:41,856 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., storeName=fbbc71fa787b8911356ef7975af46e80/C, priority=13, startTime=1731118961792; duration=0sec 2024-11-09T02:22:41,856 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:22:41,856 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbbc71fa787b8911356ef7975af46e80:C 2024-11-09T02:22:42,092 INFO [Thread-1118 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1118. 2024-11-09T02:22:42,092 INFO [Thread-1126 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1126. 2024-11-09T02:22:42,092 DEBUG [Thread-1118 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:22:42,092 DEBUG [Thread-1126 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:22:42,092 INFO [Thread-1120 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1120. 2024-11-09T02:22:42,092 DEBUG [Thread-1118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:42,092 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:42,092 DEBUG [Thread-1120 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:22:42,092 DEBUG [Thread-1120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:42,092 DEBUG [Thread-1118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:42,092 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:42,092 DEBUG [Thread-1120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:42,092 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:22:42,092 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:22:42,092 INFO [Thread-1122 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1122. 2024-11-09T02:22:42,092 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:22:42,092 DEBUG [Thread-1122 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:22:42,092 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:42,092 INFO [Thread-1124 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1124. 2024-11-09T02:22:42,092 DEBUG [Thread-1124 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:22:42,092 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:22:42,092 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:42,092 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:42,092 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:42,092 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:22:42,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T02:22:42,321 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:22:42,696 INFO [Thread-1107 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1107. 2024-11-09T02:22:42,696 INFO [Thread-1113 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1113. 2024-11-09T02:22:42,696 DEBUG [Thread-1113 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:22:42,696 DEBUG [Thread-1107 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:22:42,696 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:42,696 DEBUG [Thread-1107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:42,696 DEBUG [Thread-1107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:42,697 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:42,697 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:22:42,697 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:22:42,703 INFO [Thread-1111 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1111. 2024-11-09T02:22:42,703 INFO [Thread-1109 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1109. 2024-11-09T02:22:42,703 DEBUG [Thread-1111 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:22:42,703 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:42,703 DEBUG [Thread-1109 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:22:42,703 DEBUG [Thread-1109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:42,703 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:42,703 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:22:42,703 DEBUG [Thread-1109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:42,704 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:22:43,498 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T02:22:50,644 INFO [Thread-1115 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1115. 2024-11-09T02:22:50,645 DEBUG [Thread-1115 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:22:50,645 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:50,645 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:50,645 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:22:50,646 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T02:22:50,646 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-09T02:22:50,646 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-09T02:22:50,646 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-09T02:22:50,646 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-09T02:22:50,646 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-09T02:22:50,646 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T02:22:50,646 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4414 2024-11-09T02:22:50,646 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4180 2024-11-09T02:22:50,646 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4441 2024-11-09T02:22:50,646 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4410 2024-11-09T02:22:50,646 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4166 2024-11-09T02:22:50,646 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T02:22:50,646 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T02:22:50,646 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:22:50,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:50,647 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:50,647 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:22:50,648 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T02:22:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T02:22:50,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-09T02:22:50,652 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118970652"}]},"ts":"1731118970652"} 2024-11-09T02:22:50,654 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T02:22:50,654 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T02:22:50,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T02:22:50,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fbbc71fa787b8911356ef7975af46e80, UNASSIGN}] 2024-11-09T02:22:50,659 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fbbc71fa787b8911356ef7975af46e80, UNASSIGN 2024-11-09T02:22:50,660 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=fbbc71fa787b8911356ef7975af46e80, regionState=CLOSING, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:50,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fbbc71fa787b8911356ef7975af46e80, UNASSIGN because future has completed 2024-11-09T02:22:50,663 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T02:22:50,663 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223}] 2024-11-09T02:22:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-09T02:22:50,817 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(122): Close fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:50,817 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T02:22:50,818 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1722): Closing fbbc71fa787b8911356ef7975af46e80, disabling compactions & flushes 2024-11-09T02:22:50,818 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:50,818 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:50,818 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. after waiting 0 ms 2024-11-09T02:22:50,818 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:50,818 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(2902): Flushing fbbc71fa787b8911356ef7975af46e80 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-09T02:22:50,825 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/49076486a9db4c9984fde16a6718499c is 50, key is test_row_1/A:col10/1731118970641/Put/seqid=0 2024-11-09T02:22:50,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742140_1316 (size=9857) 2024-11-09T02:22:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-09T02:22:51,230 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/49076486a9db4c9984fde16a6718499c 2024-11-09T02:22:51,243 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/abf54dafd1b641e5b8420ce7ef9bf6fb is 50, key is test_row_1/B:col10/1731118970641/Put/seqid=0 2024-11-09T02:22:51,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742141_1317 (size=9857) 2024-11-09T02:22:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-09T02:22:51,648 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/abf54dafd1b641e5b8420ce7ef9bf6fb 2024-11-09T02:22:51,661 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/f581a229ac5945fca25d2c275db68e6c is 50, key is test_row_1/C:col10/1731118970641/Put/seqid=0 2024-11-09T02:22:51,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742142_1318 (size=9857) 2024-11-09T02:22:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-09T02:22:52,066 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/f581a229ac5945fca25d2c275db68e6c 2024-11-09T02:22:52,074 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/A/49076486a9db4c9984fde16a6718499c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/49076486a9db4c9984fde16a6718499c 2024-11-09T02:22:52,081 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/49076486a9db4c9984fde16a6718499c, entries=100, sequenceid=375, filesize=9.6 K 2024-11-09T02:22:52,082 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/B/abf54dafd1b641e5b8420ce7ef9bf6fb as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/abf54dafd1b641e5b8420ce7ef9bf6fb 2024-11-09T02:22:52,087 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/abf54dafd1b641e5b8420ce7ef9bf6fb, entries=100, sequenceid=375, filesize=9.6 K 2024-11-09T02:22:52,088 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/.tmp/C/f581a229ac5945fca25d2c275db68e6c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/f581a229ac5945fca25d2c275db68e6c 2024-11-09T02:22:52,093 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/f581a229ac5945fca25d2c275db68e6c, entries=100, sequenceid=375, filesize=9.6 K 2024-11-09T02:22:52,094 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for fbbc71fa787b8911356ef7975af46e80 in 1276ms, sequenceid=375, compaction requested=false 2024-11-09T02:22:52,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/8814a5292faa4a55a942ce48c172b3bd, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/e8a708c72be34ba4a435eb74cb39279d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/f05073feb95a42ffb3d18db3b74808f9, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/6bd0a13da40c41f1b445eaddfc9f6348, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/9b7442e7398e48ecada748063fe639bc, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/073bf17c05054caeaeee8549b6a2d999, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/2e9fa72cfa4346dd9520d19a9e72dcf8, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/621fbf339bb9461d9a3eeafba6dcc493, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/43649e5d0dae417aaa612e9c34b62b43, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/79b2f430b94d453186e0aa7942ba0940, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/f7ceb3108a6145ff9ac13e043bf5dd96, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/bc8ddf3498ad4b148d31470c4d67d016, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/df360f4d647947dbb341499ad3ab9fab, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/4d50155eb3d345d6b7160c3ede169655, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/4b2fcd88bd634cf8957d926809da8aca, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/f2df994a989b45e5a87fddb2d2d968a6, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/831672cdd8644efeae81f96e353060f2, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/747073693b40400fb2b4ccfbe0ad0a22, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/140bc6d8c1654e6e8580d112c71c6c69, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/21996ebf1cb34c1f910e03344cd0521b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/c7d2c0b470774d0199aca5b70353c0b2, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/1a9c8dad268a43889300ad33cfa68d47, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/dbf1503a001c401b8646afb02e00a862, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/9ec707a19e9c4e78855f2a4af07d3a94, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/f0db8c3a13fd458eb4800e91e4d81017] to archive 2024-11-09T02:22:52,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T02:22:52,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/8814a5292faa4a55a942ce48c172b3bd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/8814a5292faa4a55a942ce48c172b3bd 2024-11-09T02:22:52,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/e8a708c72be34ba4a435eb74cb39279d to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/e8a708c72be34ba4a435eb74cb39279d 2024-11-09T02:22:52,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/f05073feb95a42ffb3d18db3b74808f9 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/f05073feb95a42ffb3d18db3b74808f9 2024-11-09T02:22:52,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/6bd0a13da40c41f1b445eaddfc9f6348 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/6bd0a13da40c41f1b445eaddfc9f6348 2024-11-09T02:22:52,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/9b7442e7398e48ecada748063fe639bc to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/9b7442e7398e48ecada748063fe639bc 2024-11-09T02:22:52,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/073bf17c05054caeaeee8549b6a2d999 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/073bf17c05054caeaeee8549b6a2d999 2024-11-09T02:22:52,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/2e9fa72cfa4346dd9520d19a9e72dcf8 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/2e9fa72cfa4346dd9520d19a9e72dcf8 2024-11-09T02:22:52,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/621fbf339bb9461d9a3eeafba6dcc493 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/621fbf339bb9461d9a3eeafba6dcc493 2024-11-09T02:22:52,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/43649e5d0dae417aaa612e9c34b62b43 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/43649e5d0dae417aaa612e9c34b62b43 2024-11-09T02:22:52,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/79b2f430b94d453186e0aa7942ba0940 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/79b2f430b94d453186e0aa7942ba0940 2024-11-09T02:22:52,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/f7ceb3108a6145ff9ac13e043bf5dd96 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/f7ceb3108a6145ff9ac13e043bf5dd96 2024-11-09T02:22:52,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/bc8ddf3498ad4b148d31470c4d67d016 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/bc8ddf3498ad4b148d31470c4d67d016 2024-11-09T02:22:52,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/df360f4d647947dbb341499ad3ab9fab to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/df360f4d647947dbb341499ad3ab9fab 2024-11-09T02:22:52,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/4d50155eb3d345d6b7160c3ede169655 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/4d50155eb3d345d6b7160c3ede169655 2024-11-09T02:22:52,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/4b2fcd88bd634cf8957d926809da8aca to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/4b2fcd88bd634cf8957d926809da8aca 2024-11-09T02:22:52,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/f2df994a989b45e5a87fddb2d2d968a6 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/f2df994a989b45e5a87fddb2d2d968a6 2024-11-09T02:22:52,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/831672cdd8644efeae81f96e353060f2 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/831672cdd8644efeae81f96e353060f2 2024-11-09T02:22:52,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/747073693b40400fb2b4ccfbe0ad0a22 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/747073693b40400fb2b4ccfbe0ad0a22 2024-11-09T02:22:52,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/140bc6d8c1654e6e8580d112c71c6c69 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/140bc6d8c1654e6e8580d112c71c6c69 2024-11-09T02:22:52,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/21996ebf1cb34c1f910e03344cd0521b to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/21996ebf1cb34c1f910e03344cd0521b 2024-11-09T02:22:52,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/c7d2c0b470774d0199aca5b70353c0b2 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/c7d2c0b470774d0199aca5b70353c0b2 2024-11-09T02:22:52,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/1a9c8dad268a43889300ad33cfa68d47 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/1a9c8dad268a43889300ad33cfa68d47 2024-11-09T02:22:52,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/dbf1503a001c401b8646afb02e00a862 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/dbf1503a001c401b8646afb02e00a862 2024-11-09T02:22:52,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/9ec707a19e9c4e78855f2a4af07d3a94 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/9ec707a19e9c4e78855f2a4af07d3a94 2024-11-09T02:22:52,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/f0db8c3a13fd458eb4800e91e4d81017 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/f0db8c3a13fd458eb4800e91e4d81017 2024-11-09T02:22:52,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/f46c9f5291da47a0bc4e475a0400817d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/2eb5de7a6d80499ba384983328d5f7a5, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/594a8d05a8c9422da679bc1a1ac1ed5e, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/e4c6a2a731ff451e92afd84deb5dad8a, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/e245253a16794653a06babc2bf06d229, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/c55374cde69a4942a9cd105c33329fab, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/95f3fc3376dd4da19c7c3711d3fdeb1d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/2ae5c6325d0640678582259c0da56ef7, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/46176b8aed084053a738ac161ae73063, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/cdca675112a24502ad3533328959efbf, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/dd85ba50eeba46efb9e9354fbc47db57, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/26f5309c4a0e47469cd29bb924db9f96, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/6227dd0788f841428b761a7fbf61bcd7, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/41757ec5d1cf4951b6b9fb6d45231dbc, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/7b32a76aa6e64b6caf5ca93955d97026, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/111b2d673c594c069ec14ad20593efff, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/0e9e94dae297488a81d901741f4ca5e0, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/9166b799c69b4f638a434aef868bf0e6, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/1d084ab4a12f4aac87c8e3fa803076ab, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/dde13a5c48924c0c9f0b814e5c60ccc1, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/37d1d4cca7de4dfda17ba8bcbd3f987b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/5fb8514177e94567b9bb5a795b8d46c4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/6e0218f28cce49babeae5e012e371e9b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/de053b937b25483aa3f6831ae6a12917, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/9fbc5437e0f0413b9cb5a0a9bbaab1d1] to archive 2024-11-09T02:22:52,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T02:22:52,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/f46c9f5291da47a0bc4e475a0400817d to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/f46c9f5291da47a0bc4e475a0400817d 2024-11-09T02:22:52,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/2eb5de7a6d80499ba384983328d5f7a5 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/2eb5de7a6d80499ba384983328d5f7a5 2024-11-09T02:22:52,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/594a8d05a8c9422da679bc1a1ac1ed5e to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/594a8d05a8c9422da679bc1a1ac1ed5e 2024-11-09T02:22:52,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/e4c6a2a731ff451e92afd84deb5dad8a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/e4c6a2a731ff451e92afd84deb5dad8a 2024-11-09T02:22:52,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/e245253a16794653a06babc2bf06d229 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/e245253a16794653a06babc2bf06d229 2024-11-09T02:22:52,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/c55374cde69a4942a9cd105c33329fab to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/c55374cde69a4942a9cd105c33329fab 2024-11-09T02:22:52,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/95f3fc3376dd4da19c7c3711d3fdeb1d to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/95f3fc3376dd4da19c7c3711d3fdeb1d 2024-11-09T02:22:52,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/2ae5c6325d0640678582259c0da56ef7 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/2ae5c6325d0640678582259c0da56ef7 2024-11-09T02:22:52,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/46176b8aed084053a738ac161ae73063 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/46176b8aed084053a738ac161ae73063 2024-11-09T02:22:52,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/cdca675112a24502ad3533328959efbf to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/cdca675112a24502ad3533328959efbf 2024-11-09T02:22:52,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/dd85ba50eeba46efb9e9354fbc47db57 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/dd85ba50eeba46efb9e9354fbc47db57 2024-11-09T02:22:52,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/26f5309c4a0e47469cd29bb924db9f96 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/26f5309c4a0e47469cd29bb924db9f96 2024-11-09T02:22:52,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/6227dd0788f841428b761a7fbf61bcd7 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/6227dd0788f841428b761a7fbf61bcd7 2024-11-09T02:22:52,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/41757ec5d1cf4951b6b9fb6d45231dbc to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/41757ec5d1cf4951b6b9fb6d45231dbc 2024-11-09T02:22:52,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/7b32a76aa6e64b6caf5ca93955d97026 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/7b32a76aa6e64b6caf5ca93955d97026 2024-11-09T02:22:52,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/111b2d673c594c069ec14ad20593efff to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/111b2d673c594c069ec14ad20593efff 2024-11-09T02:22:52,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/0e9e94dae297488a81d901741f4ca5e0 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/0e9e94dae297488a81d901741f4ca5e0 2024-11-09T02:22:52,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/9166b799c69b4f638a434aef868bf0e6 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/9166b799c69b4f638a434aef868bf0e6 2024-11-09T02:22:52,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/1d084ab4a12f4aac87c8e3fa803076ab to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/1d084ab4a12f4aac87c8e3fa803076ab 2024-11-09T02:22:52,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/dde13a5c48924c0c9f0b814e5c60ccc1 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/dde13a5c48924c0c9f0b814e5c60ccc1 2024-11-09T02:22:52,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/37d1d4cca7de4dfda17ba8bcbd3f987b to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/37d1d4cca7de4dfda17ba8bcbd3f987b 2024-11-09T02:22:52,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/5fb8514177e94567b9bb5a795b8d46c4 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/5fb8514177e94567b9bb5a795b8d46c4 2024-11-09T02:22:52,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/6e0218f28cce49babeae5e012e371e9b to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/6e0218f28cce49babeae5e012e371e9b 2024-11-09T02:22:52,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/de053b937b25483aa3f6831ae6a12917 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/de053b937b25483aa3f6831ae6a12917 2024-11-09T02:22:52,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/9fbc5437e0f0413b9cb5a0a9bbaab1d1 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/9fbc5437e0f0413b9cb5a0a9bbaab1d1 2024-11-09T02:22:52,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/9b216fc7c2d748bca9b97c624fd23526, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/d1f3b0e83a2343d2a8243cd47dd72e9b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/59dde14b2dd444438283a11590030afb, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/13b2310422094204a8cd9253620d4da4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/97e5029bb72e428a80bae35904d61568, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/2510b59bbd054c6a847fadd3c44c2520, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/17bf9b4c2bc44c2b971bc1dc4acbbf0e, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/9d6536d000754a5a964a9b0b1381ae6b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/479b4f88c8fa487fb1561779294420eb, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/2824c221be86420798fb4660847d06ba, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/f2c4399a85774b16badc9dcb4039a384, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/e02215022b9d4ebda06e4fa371fac865, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/8b52f2650af74532b13ff33b79482bb8, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/e7fa25ace5c94befa875da9952532680, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/193a53e2091a414a831a3e4a7a039e31, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/2a2324aab8694905abf718d5e47f70e5, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/c86ae453c79a49829958862d0e241a93, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/8e987d1d382044f6aaa263c44e12db10, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/b671cd1f7a5d40ab885ba705481ccdfe, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/71bfc22acc864caa9408485aed212ad1, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/8e076111a75a4f0b8668014efa69d051, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/3349acf14fc7429898c4c8c4656580dd, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/746e634d67da40c6bcc8fb8eeca8da51, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/1dd772b876db4045b518a416248904b8, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/c879deb3c7e14898bd1c65a35a253b42] to archive 2024-11-09T02:22:52,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T02:22:52,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/9b216fc7c2d748bca9b97c624fd23526 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/9b216fc7c2d748bca9b97c624fd23526 2024-11-09T02:22:52,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/d1f3b0e83a2343d2a8243cd47dd72e9b to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/d1f3b0e83a2343d2a8243cd47dd72e9b 2024-11-09T02:22:52,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/59dde14b2dd444438283a11590030afb to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/59dde14b2dd444438283a11590030afb 2024-11-09T02:22:52,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/13b2310422094204a8cd9253620d4da4 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/13b2310422094204a8cd9253620d4da4 2024-11-09T02:22:52,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/97e5029bb72e428a80bae35904d61568 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/97e5029bb72e428a80bae35904d61568 2024-11-09T02:22:52,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/2510b59bbd054c6a847fadd3c44c2520 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/2510b59bbd054c6a847fadd3c44c2520 2024-11-09T02:22:52,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/17bf9b4c2bc44c2b971bc1dc4acbbf0e to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/17bf9b4c2bc44c2b971bc1dc4acbbf0e 2024-11-09T02:22:52,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/9d6536d000754a5a964a9b0b1381ae6b to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/9d6536d000754a5a964a9b0b1381ae6b 2024-11-09T02:22:52,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/479b4f88c8fa487fb1561779294420eb to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/479b4f88c8fa487fb1561779294420eb 2024-11-09T02:22:52,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/2824c221be86420798fb4660847d06ba to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/2824c221be86420798fb4660847d06ba 2024-11-09T02:22:52,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/f2c4399a85774b16badc9dcb4039a384 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/f2c4399a85774b16badc9dcb4039a384 2024-11-09T02:22:52,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/e02215022b9d4ebda06e4fa371fac865 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/e02215022b9d4ebda06e4fa371fac865 2024-11-09T02:22:52,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/8b52f2650af74532b13ff33b79482bb8 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/8b52f2650af74532b13ff33b79482bb8 2024-11-09T02:22:52,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/e7fa25ace5c94befa875da9952532680 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/e7fa25ace5c94befa875da9952532680 2024-11-09T02:22:52,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/193a53e2091a414a831a3e4a7a039e31 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/193a53e2091a414a831a3e4a7a039e31 2024-11-09T02:22:52,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/2a2324aab8694905abf718d5e47f70e5 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/2a2324aab8694905abf718d5e47f70e5 2024-11-09T02:22:52,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/c86ae453c79a49829958862d0e241a93 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/c86ae453c79a49829958862d0e241a93 2024-11-09T02:22:52,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/8e987d1d382044f6aaa263c44e12db10 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/8e987d1d382044f6aaa263c44e12db10 2024-11-09T02:22:52,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/b671cd1f7a5d40ab885ba705481ccdfe to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/b671cd1f7a5d40ab885ba705481ccdfe 2024-11-09T02:22:52,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/71bfc22acc864caa9408485aed212ad1 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/71bfc22acc864caa9408485aed212ad1 2024-11-09T02:22:52,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/8e076111a75a4f0b8668014efa69d051 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/8e076111a75a4f0b8668014efa69d051 2024-11-09T02:22:52,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/3349acf14fc7429898c4c8c4656580dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/3349acf14fc7429898c4c8c4656580dd 2024-11-09T02:22:52,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/746e634d67da40c6bcc8fb8eeca8da51 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/746e634d67da40c6bcc8fb8eeca8da51 2024-11-09T02:22:52,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/1dd772b876db4045b518a416248904b8 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/1dd772b876db4045b518a416248904b8 2024-11-09T02:22:52,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/c879deb3c7e14898bd1c65a35a253b42 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/c879deb3c7e14898bd1c65a35a253b42 2024-11-09T02:22:52,172 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/recovered.edits/378.seqid, newMaxSeqId=378, maxSeqId=1 2024-11-09T02:22:52,172 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. 2024-11-09T02:22:52,172 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1676): Region close journal for fbbc71fa787b8911356ef7975af46e80: Waiting for close lock at 1731118970818Running coprocessor pre-close hooks at 1731118970818Disabling compacts and flushes for region at 1731118970818Disabling writes for close at 1731118970818Obtaining lock to block concurrent updates at 1731118970818Preparing flush snapshotting stores in fbbc71fa787b8911356ef7975af46e80 at 1731118970818Finished memstore snapshotting TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80., syncing WAL and waiting on mvcc, flushsize=dataSize=41220, getHeapSize=108720, getOffHeapSize=0, getCellsCount=900 at 1731118970819 (+1 ms)Flushing stores of TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80. at 1731118970821 (+2 ms)Flushing fbbc71fa787b8911356ef7975af46e80/A: creating writer at 1731118970821Flushing fbbc71fa787b8911356ef7975af46e80/A: appending metadata at 1731118970825 (+4 ms)Flushing fbbc71fa787b8911356ef7975af46e80/A: closing flushed file at 1731118970825Flushing fbbc71fa787b8911356ef7975af46e80/B: creating writer at 1731118971240 (+415 ms)Flushing fbbc71fa787b8911356ef7975af46e80/B: appending metadata at 1731118971243 (+3 ms)Flushing fbbc71fa787b8911356ef7975af46e80/B: closing flushed file at 1731118971243Flushing fbbc71fa787b8911356ef7975af46e80/C: creating writer at 1731118971655 (+412 ms)Flushing fbbc71fa787b8911356ef7975af46e80/C: appending metadata at 1731118971661 (+6 ms)Flushing fbbc71fa787b8911356ef7975af46e80/C: closing flushed file at 1731118971661Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a7ca8c3: reopening flushed file at 1731118972073 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4cb422d6: reopening flushed file at 1731118972081 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1041c871: reopening flushed file at 1731118972087 (+6 ms)Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for fbbc71fa787b8911356ef7975af46e80 in 1276ms, sequenceid=375, compaction requested=false at 1731118972094 (+7 ms)Writing region close event to WAL at 1731118972169 (+75 ms)Running coprocessor post-close hooks at 1731118972172 (+3 ms)Closed at 1731118972172 2024-11-09T02:22:52,174 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(157): Closed fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:52,174 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=fbbc71fa787b8911356ef7975af46e80, regionState=CLOSED 2024-11-09T02:22:52,175 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 because future has completed 2024-11-09T02:22:52,177 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-09T02:22:52,177 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseRegionProcedure fbbc71fa787b8911356ef7975af46e80, server=53f286d3af3b,46079,1731118875223 in 1.5130 sec 2024-11-09T02:22:52,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-09T02:22:52,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fbbc71fa787b8911356ef7975af46e80, UNASSIGN in 1.5200 sec 2024-11-09T02:22:52,180 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-09T02:22:52,180 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5240 sec 2024-11-09T02:22:52,181 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118972181"}]},"ts":"1731118972181"} 2024-11-09T02:22:52,182 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T02:22:52,182 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T02:22:52,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5350 sec 2024-11-09T02:22:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-09T02:22:52,801 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:22:52,801 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T02:22:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:22:52,803 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:22:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-09T02:22:52,803 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=82, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:22:52,805 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:52,807 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A, FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B, FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C, FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/recovered.edits] 2024-11-09T02:22:52,809 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/49076486a9db4c9984fde16a6718499c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/49076486a9db4c9984fde16a6718499c 2024-11-09T02:22:52,811 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/dc7b7edbc24946f5a221bdb36c00bca8 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/A/dc7b7edbc24946f5a221bdb36c00bca8 2024-11-09T02:22:52,813 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/422bbc9b8d8d42df8ffef53d85bac646 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/422bbc9b8d8d42df8ffef53d85bac646 2024-11-09T02:22:52,814 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/abf54dafd1b641e5b8420ce7ef9bf6fb to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/B/abf54dafd1b641e5b8420ce7ef9bf6fb 2024-11-09T02:22:52,815 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/c0d634a1d96d4d07a9ee2c80280a4e02 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/c0d634a1d96d4d07a9ee2c80280a4e02 2024-11-09T02:22:52,816 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/f581a229ac5945fca25d2c275db68e6c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/C/f581a229ac5945fca25d2c275db68e6c 2024-11-09T02:22:52,818 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/recovered.edits/378.seqid to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80/recovered.edits/378.seqid 2024-11-09T02:22:52,819 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/fbbc71fa787b8911356ef7975af46e80 2024-11-09T02:22:52,819 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T02:22:52,820 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=82, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:22:52,824 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T02:22:52,826 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T02:22:52,827 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=82, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:22:52,827 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T02:22:52,827 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731118972827"}]},"ts":"9223372036854775807"} 2024-11-09T02:22:52,829 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T02:22:52,829 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fbbc71fa787b8911356ef7975af46e80, NAME => 'TestAcidGuarantees,,1731118939880.fbbc71fa787b8911356ef7975af46e80.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T02:22:52,829 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T02:22:52,829 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731118972829"}]},"ts":"9223372036854775807"} 2024-11-09T02:22:52,830 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T02:22:52,831 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=82, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:22:52,832 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 30 msec 2024-11-09T02:22:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-09T02:22:52,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T02:22:52,910 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:22:52,925 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=236 (was 238), OpenFileDescriptor=445 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=447 (was 525), ProcessCount=11 (was 11), AvailableMemoryMB=2968 (was 2884) - AvailableMemoryMB LEAK? - 2024-11-09T02:22:52,934 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=236, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=447, ProcessCount=11, AvailableMemoryMB=2968 2024-11-09T02:22:52,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T02:22:52,936 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:22:52,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T02:22:52,984 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:22:52,985 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:22:52,985 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 83 2024-11-09T02:22:52,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-09T02:22:52,987 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:22:52,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742143_1319 (size=959) 2024-11-09T02:22:53,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-09T02:22:53,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-09T02:22:53,404 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a 2024-11-09T02:22:53,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742144_1320 (size=53) 2024-11-09T02:22:53,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-09T02:22:53,814 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:22:53,815 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a695692878ab8811a17602b58ad8f4dd, disabling compactions & flushes 2024-11-09T02:22:53,815 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:22:53,815 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:22:53,815 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. after waiting 0 ms 2024-11-09T02:22:53,815 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:22:53,815 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:22:53,815 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a695692878ab8811a17602b58ad8f4dd: Waiting for close lock at 1731118973815Disabling compacts and flushes for region at 1731118973815Disabling writes for close at 1731118973815Writing region close event to WAL at 1731118973815Closed at 1731118973815 2024-11-09T02:22:53,818 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:22:53,819 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731118973818"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118973818"}]},"ts":"1731118973818"} 2024-11-09T02:22:53,822 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T02:22:53,824 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:22:53,825 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118973824"}]},"ts":"1731118973824"} 2024-11-09T02:22:53,827 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T02:22:53,827 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a695692878ab8811a17602b58ad8f4dd, ASSIGN}] 2024-11-09T02:22:53,828 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a695692878ab8811a17602b58ad8f4dd, ASSIGN 2024-11-09T02:22:53,829 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a695692878ab8811a17602b58ad8f4dd, ASSIGN; state=OFFLINE, location=53f286d3af3b,46079,1731118875223; forceNewPlan=false, retain=false 2024-11-09T02:22:53,980 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=a695692878ab8811a17602b58ad8f4dd, regionState=OPENING, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:53,984 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a695692878ab8811a17602b58ad8f4dd, ASSIGN because future has completed 2024-11-09T02:22:53,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223}] 2024-11-09T02:22:54,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-09T02:22:54,148 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:22:54,148 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7752): Opening region: {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:22:54,149 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:54,149 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:22:54,150 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7794): checking encryption for a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:54,150 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7797): checking classloading for a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:54,152 INFO [StoreOpener-a695692878ab8811a17602b58ad8f4dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:54,154 INFO [StoreOpener-a695692878ab8811a17602b58ad8f4dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a695692878ab8811a17602b58ad8f4dd columnFamilyName A 2024-11-09T02:22:54,154 DEBUG [StoreOpener-a695692878ab8811a17602b58ad8f4dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:22:54,155 INFO [StoreOpener-a695692878ab8811a17602b58ad8f4dd-1 {}] regionserver.HStore(327): Store=a695692878ab8811a17602b58ad8f4dd/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:22:54,155 INFO [StoreOpener-a695692878ab8811a17602b58ad8f4dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:54,157 INFO [StoreOpener-a695692878ab8811a17602b58ad8f4dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a695692878ab8811a17602b58ad8f4dd columnFamilyName B 2024-11-09T02:22:54,157 DEBUG [StoreOpener-a695692878ab8811a17602b58ad8f4dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:22:54,158 INFO [StoreOpener-a695692878ab8811a17602b58ad8f4dd-1 {}] regionserver.HStore(327): Store=a695692878ab8811a17602b58ad8f4dd/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:22:54,158 INFO [StoreOpener-a695692878ab8811a17602b58ad8f4dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:54,160 INFO [StoreOpener-a695692878ab8811a17602b58ad8f4dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a695692878ab8811a17602b58ad8f4dd columnFamilyName C 2024-11-09T02:22:54,160 DEBUG [StoreOpener-a695692878ab8811a17602b58ad8f4dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:22:54,160 INFO [StoreOpener-a695692878ab8811a17602b58ad8f4dd-1 {}] regionserver.HStore(327): Store=a695692878ab8811a17602b58ad8f4dd/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:22:54,161 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1038): replaying wal for a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:54,162 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:54,162 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:54,165 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1048): stopping wal replay for a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:54,165 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1060): Cleaning up temporary data for a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:54,165 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T02:22:54,167 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1093): writing seq id for a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:54,169 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:22:54,169 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1114): Opened a695692878ab8811a17602b58ad8f4dd; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73780510, jitterRate=0.09941527247428894}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T02:22:54,169 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:54,170 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1006): Region open journal for a695692878ab8811a17602b58ad8f4dd: Running coprocessor pre-open hook at 1731118974150Writing region info on filesystem at 1731118974150Initializing all the Stores at 1731118974151 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118974151Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118974152 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118974152Cleaning up temporary data from old regions at 1731118974165 (+13 ms)Running coprocessor post-open hooks at 1731118974169 (+4 ms)Region opened successfully at 1731118974170 (+1 ms) 2024-11-09T02:22:54,170 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., pid=85, masterSystemTime=1731118974140 2024-11-09T02:22:54,172 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:22:54,172 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:22:54,172 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=a695692878ab8811a17602b58ad8f4dd, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:54,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 because future has completed 2024-11-09T02:22:54,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-09T02:22:54,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 in 189 msec 2024-11-09T02:22:54,177 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-09T02:22:54,177 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a695692878ab8811a17602b58ad8f4dd, ASSIGN in 349 msec 2024-11-09T02:22:54,178 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:22:54,178 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118974178"}]},"ts":"1731118974178"} 2024-11-09T02:22:54,179 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T02:22:54,180 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:22:54,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2440 sec 2024-11-09T02:22:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-09T02:22:55,141 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:22:55,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af6f666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:55,144 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:22:55,144 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:22:55,147 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:22:55,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:22:55,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:22:55,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@375c6607, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:22:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:22:55,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:55,150 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:22:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e436efb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:55,151 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:22:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:22:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:22:55,153 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:22:55,153 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T02:22:55,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 53f286d3af3b,45041,1731118874457 2024-11-09T02:22:55,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3216b291 2024-11-09T02:22:55,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T02:22:55,155 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T02:22:55,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T02:22:55,156 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:22:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-09T02:22:55,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742145_1321 (size=995) 2024-11-09T02:22:55,571 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-09T02:22:55,571 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-09T02:22:55,577 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T02:22:55,581 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a695692878ab8811a17602b58ad8f4dd, REOPEN/MOVE}] 2024-11-09T02:22:55,582 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a695692878ab8811a17602b58ad8f4dd, REOPEN/MOVE 2024-11-09T02:22:55,583 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=a695692878ab8811a17602b58ad8f4dd, regionState=CLOSING, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:55,584 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a695692878ab8811a17602b58ad8f4dd, REOPEN/MOVE because future has completed 2024-11-09T02:22:55,584 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T02:22:55,584 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223}] 2024-11-09T02:22:55,738 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:55,738 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T02:22:55,738 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing a695692878ab8811a17602b58ad8f4dd, disabling compactions & flushes 2024-11-09T02:22:55,738 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:22:55,739 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:22:55,739 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. after waiting 0 ms 2024-11-09T02:22:55,739 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:22:55,748 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:22:55,749 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:22:55,749 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for a695692878ab8811a17602b58ad8f4dd: Waiting for close lock at 1731118975738Running coprocessor pre-close hooks at 1731118975738Disabling compacts and flushes for region at 1731118975738Disabling writes for close at 1731118975739 (+1 ms)Writing region close event to WAL at 1731118975740 (+1 ms)Running coprocessor post-close hooks at 1731118975749 (+9 ms)Closed at 1731118975749 2024-11-09T02:22:55,749 WARN [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionServer(3299): Not adding moved region record: a695692878ab8811a17602b58ad8f4dd to self. 2024-11-09T02:22:55,753 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:55,753 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=a695692878ab8811a17602b58ad8f4dd, regionState=CLOSED 2024-11-09T02:22:55,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 because future has completed 2024-11-09T02:22:55,759 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-09T02:22:55,759 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 in 172 msec 2024-11-09T02:22:55,760 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a695692878ab8811a17602b58ad8f4dd, REOPEN/MOVE; state=CLOSED, location=53f286d3af3b,46079,1731118875223; forceNewPlan=false, retain=true 2024-11-09T02:22:55,910 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=a695692878ab8811a17602b58ad8f4dd, regionState=OPENING, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:55,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a695692878ab8811a17602b58ad8f4dd, REOPEN/MOVE because future has completed 2024-11-09T02:22:55,913 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223}] 2024-11-09T02:22:56,075 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:22:56,075 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7752): Opening region: {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:22:56,076 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:56,076 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:22:56,076 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7794): checking encryption for a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:56,076 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7797): checking classloading for a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:56,077 INFO [StoreOpener-a695692878ab8811a17602b58ad8f4dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:56,078 INFO [StoreOpener-a695692878ab8811a17602b58ad8f4dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a695692878ab8811a17602b58ad8f4dd columnFamilyName A 2024-11-09T02:22:56,079 DEBUG [StoreOpener-a695692878ab8811a17602b58ad8f4dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:22:56,079 INFO [StoreOpener-a695692878ab8811a17602b58ad8f4dd-1 {}] regionserver.HStore(327): Store=a695692878ab8811a17602b58ad8f4dd/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:22:56,079 INFO [StoreOpener-a695692878ab8811a17602b58ad8f4dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:56,080 INFO [StoreOpener-a695692878ab8811a17602b58ad8f4dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a695692878ab8811a17602b58ad8f4dd columnFamilyName B 2024-11-09T02:22:56,080 DEBUG [StoreOpener-a695692878ab8811a17602b58ad8f4dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:22:56,081 INFO [StoreOpener-a695692878ab8811a17602b58ad8f4dd-1 {}] regionserver.HStore(327): Store=a695692878ab8811a17602b58ad8f4dd/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:22:56,081 INFO [StoreOpener-a695692878ab8811a17602b58ad8f4dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:56,081 INFO [StoreOpener-a695692878ab8811a17602b58ad8f4dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a695692878ab8811a17602b58ad8f4dd columnFamilyName C 2024-11-09T02:22:56,081 DEBUG [StoreOpener-a695692878ab8811a17602b58ad8f4dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:22:56,082 INFO [StoreOpener-a695692878ab8811a17602b58ad8f4dd-1 {}] regionserver.HStore(327): Store=a695692878ab8811a17602b58ad8f4dd/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:22:56,082 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1038): replaying wal for a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:56,083 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:56,083 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:56,084 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1048): stopping wal replay for a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:56,084 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1060): Cleaning up temporary data for a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:56,085 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T02:22:56,086 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1093): writing seq id for a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:56,087 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1114): Opened a695692878ab8811a17602b58ad8f4dd; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62085201, jitterRate=-0.0748584121465683}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T02:22:56,087 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:56,088 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1006): Region open journal for a695692878ab8811a17602b58ad8f4dd: Running coprocessor pre-open hook at 1731118976076Writing region info on filesystem at 1731118976076Initializing all the Stores at 1731118976077 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118976077Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118976077Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118976077Cleaning up temporary data from old regions at 1731118976084 (+7 ms)Running coprocessor post-open hooks at 1731118976087 (+3 ms)Region opened successfully at 1731118976087 2024-11-09T02:22:56,088 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., pid=90, masterSystemTime=1731118976066 2024-11-09T02:22:56,090 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:22:56,090 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:22:56,091 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=a695692878ab8811a17602b58ad8f4dd, regionState=OPEN, openSeqNum=5, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:56,092 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 because future has completed 2024-11-09T02:22:56,095 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=88 2024-11-09T02:22:56,095 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 in 180 msec 2024-11-09T02:22:56,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-09T02:22:56,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a695692878ab8811a17602b58ad8f4dd, REOPEN/MOVE in 514 msec 2024-11-09T02:22:56,099 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-09T02:22:56,099 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 520 msec 2024-11-09T02:22:56,102 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 944 msec 2024-11-09T02:22:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-09T02:22:56,103 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:22:56,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ff38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,103 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:22:56,103 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:22:56,104 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:22:56,104 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:22:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:22:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66f93911, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:22:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:22:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:56,106 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:22:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d9398b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,108 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,108 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:22:56,109 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:22:56,110 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:22:56,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:22:56,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:22:56,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@129dfbb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:22:56,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:22:56,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:56,111 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:22:56,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@237f38cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,112 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,112 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:22:56,112 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:22:56,113 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:22:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:22:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:22:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76e9390a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:22:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:22:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:56,114 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:22:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e63f936, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,115 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:22:56,115 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:22:56,116 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:22:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:22:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:22:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e0bf103, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:22:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:22:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:56,117 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:22:56,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1be3998f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,118 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:22:56,118 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:22:56,119 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:22:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:22:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:22:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30062e19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:22:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:22:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:56,120 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:22:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68e7e9ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,121 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,121 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:22:56,121 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:22:56,122 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:22:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:22:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:22:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f5f3262, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:22:56,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:22:56,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:56,123 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:22:56,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ed3a255, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,125 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,125 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:22:56,125 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:22:56,126 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:22:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:22:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:22:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e6db39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:22:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:22:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:56,128 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:22:56,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eace9b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,129 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:22:56,129 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:22:56,130 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:22:56,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:22:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:22:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aa0f48a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:22:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:22:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:56,132 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:22:56,132 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bcf36a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,133 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:22:56,133 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:22:56,134 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:22:56,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:22:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:22:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3def8f2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:22:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:22:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:56,136 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:22:56,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41cde98b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,137 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,137 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:22:56,137 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:22:56,137 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:22:56,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:22:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:22:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50f9b80f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:22:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:22:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:22:56,139 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:22:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36f47d55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:22:56,140 DEBUG [Thread-1495 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:22:56,140 DEBUG [Thread-1489 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:22:56,140 DEBUG [Thread-1476 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:22:56,140 DEBUG [Thread-1480 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:22:56,140 DEBUG [Thread-1493 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:22:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:22:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:22:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:22:56,140 DEBUG [Thread-1484 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:22:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:22:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:22:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:22:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:22:56,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:22:56,141 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:22:56,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:22:56,141 DEBUG [Thread-1491 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:22:56,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:22:56,142 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:22:56,142 DEBUG [Thread-1478 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:22:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:22:56,142 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:22:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:22:56,142 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:22:56,142 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:22:56,142 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:22:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-09T02:22:56,143 DEBUG [Thread-1482 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:22:56,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:22:56,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:22:56,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:22:56,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:22:56,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:22:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:22:56,145 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:22:56,145 DEBUG [Thread-1487 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:22:56,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-09T02:22:56,145 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:22:56,146 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:22:56,146 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:22:56,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:22:56,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:22:56,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:22:56,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:22:56,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:22:56,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:22:56,147 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:22:56,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:22:56,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:22:56,149 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:22:56,149 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:22:56,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:22:56,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:22:56,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:22:56,151 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:22:56,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:22:56,157 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:22:56,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a695692878ab8811a17602b58ad8f4dd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T02:22:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:56,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099de24fb689014107b9d3425ff1200463_a695692878ab8811a17602b58ad8f4dd is 50, key is test_row_0/A:col10/1731118976154/Put/seqid=0 2024-11-09T02:22:56,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742146_1322 (size=14594) 2024-11-09T02:22:56,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119036188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:56,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:56,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119036190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:56,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:56,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:56,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:56,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119036194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:56,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:56,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:56,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119036195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:56,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:56,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119036196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:56,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:56,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-09T02:22:56,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:56,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-09T02:22:56,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119036300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:56,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119036300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:56,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:22:56,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:22:56,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:22:56,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:56,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:56,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:56,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:56,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119036310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:56,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:56,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119036310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:56,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:56,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:56,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119036311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:56,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:56,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-09T02:22:56,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-09T02:22:56,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:22:56,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:22:56,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:22:56,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:56,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:56,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:56,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:56,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119036510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:56,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:56,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119036520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119036520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119036521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:56,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119036530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:56,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:56,581 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:22:56,585 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099de24fb689014107b9d3425ff1200463_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099de24fb689014107b9d3425ff1200463_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:56,586 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/74b4ce6bbde54972af252da329afa084, store: [table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:22:56,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/74b4ce6bbde54972af252da329afa084 is 175, key is test_row_0/A:col10/1731118976154/Put/seqid=0 2024-11-09T02:22:56,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742147_1323 (size=39549) 2024-11-09T02:22:56,590 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/74b4ce6bbde54972af252da329afa084 2024-11-09T02:22:56,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/b24c730f62a643b5a9c23acd9ba14919 is 50, key is test_row_0/B:col10/1731118976154/Put/seqid=0 2024-11-09T02:22:56,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-09T02:22:56,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:22:56,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:22:56,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:22:56,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:56,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:56,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:56,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742148_1324 (size=12001) 2024-11-09T02:22:56,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/b24c730f62a643b5a9c23acd9ba14919 2024-11-09T02:22:56,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/9e14c1cfa9f04886a6525830711da1e8 is 50, key is test_row_0/C:col10/1731118976154/Put/seqid=0 2024-11-09T02:22:56,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742149_1325 (size=12001) 2024-11-09T02:22:56,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-09T02:22:56,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:22:56,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:22:56,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:22:56,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:56,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:56,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-09T02:22:56,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:56,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119036830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:56,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:56,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:56,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119036830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:56,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:56,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119036830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:56,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:56,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119036839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:56,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:56,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:56,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119036840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:56,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:56,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:56,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-09T02:22:56,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:22:56,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:22:56,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:22:56,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:56,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:22:57,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/9e14c1cfa9f04886a6525830711da1e8 2024-11-09T02:22:57,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/74b4ce6bbde54972af252da329afa084 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/74b4ce6bbde54972af252da329afa084 2024-11-09T02:22:57,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/74b4ce6bbde54972af252da329afa084, entries=200, sequenceid=15, filesize=38.6 K 2024-11-09T02:22:57,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/b24c730f62a643b5a9c23acd9ba14919 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/b24c730f62a643b5a9c23acd9ba14919 2024-11-09T02:22:57,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/b24c730f62a643b5a9c23acd9ba14919, entries=150, sequenceid=15, filesize=11.7 K 2024-11-09T02:22:57,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/9e14c1cfa9f04886a6525830711da1e8 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/9e14c1cfa9f04886a6525830711da1e8 2024-11-09T02:22:57,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/9e14c1cfa9f04886a6525830711da1e8, entries=150, sequenceid=15, filesize=11.7 K 2024-11-09T02:22:57,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a695692878ab8811a17602b58ad8f4dd in 905ms, sequenceid=15, compaction requested=false 2024-11-09T02:22:57,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:22:57,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-09T02:22:57,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:22:57,073 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing a695692878ab8811a17602b58ad8f4dd 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T02:22:57,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091a32b92ff07f4ca09a2604f9466c52d0_a695692878ab8811a17602b58ad8f4dd is 50, key is test_row_0/A:col10/1731118976194/Put/seqid=0 2024-11-09T02:22:57,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742150_1326 (size=12154) 2024-11-09T02:22:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-09T02:22:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:57,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:22:57,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119037352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:57,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119037354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:57,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:57,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119037356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:57,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:57,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119037356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:57,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:57,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119037360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:57,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:57,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119037460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:57,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:57,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119037470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119037470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:57,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119037470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:57,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:22:57,488 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091a32b92ff07f4ca09a2604f9466c52d0_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091a32b92ff07f4ca09a2604f9466c52d0_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:57,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/8a6d0d322bdb495f9c6732027dbf6e03, store: [table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:22:57,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/8a6d0d322bdb495f9c6732027dbf6e03 is 175, key is test_row_0/A:col10/1731118976194/Put/seqid=0 2024-11-09T02:22:57,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742151_1327 (size=30955) 2024-11-09T02:22:57,493 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/8a6d0d322bdb495f9c6732027dbf6e03 2024-11-09T02:22:57,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/7b6a921ee6a840d186a8a30a8e280041 is 50, key is test_row_0/B:col10/1731118976194/Put/seqid=0 2024-11-09T02:22:57,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742152_1328 (size=12001) 2024-11-09T02:22:57,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:57,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119037670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:57,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:57,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119037680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:57,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:57,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119037680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:57,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:57,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119037680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:57,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:57,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:57,699 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:22:57,906 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/7b6a921ee6a840d186a8a30a8e280041 2024-11-09T02:22:57,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/57c2ceddf3894d00b2488174830077f8 is 50, key is test_row_0/C:col10/1731118976194/Put/seqid=0 2024-11-09T02:22:57,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742153_1329 (size=12001) 2024-11-09T02:22:57,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:57,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119037980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:57,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:57,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:57,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:57,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119037990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:57,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119037990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:57,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:57,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119037991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-09T02:22:58,317 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/57c2ceddf3894d00b2488174830077f8 2024-11-09T02:22:58,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/8a6d0d322bdb495f9c6732027dbf6e03 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/8a6d0d322bdb495f9c6732027dbf6e03 2024-11-09T02:22:58,379 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/8a6d0d322bdb495f9c6732027dbf6e03, entries=150, sequenceid=40, filesize=30.2 K 2024-11-09T02:22:58,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/7b6a921ee6a840d186a8a30a8e280041 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/7b6a921ee6a840d186a8a30a8e280041 2024-11-09T02:22:58,383 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/7b6a921ee6a840d186a8a30a8e280041, entries=150, sequenceid=40, filesize=11.7 K 2024-11-09T02:22:58,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/57c2ceddf3894d00b2488174830077f8 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/57c2ceddf3894d00b2488174830077f8 2024-11-09T02:22:58,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:58,387 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/57c2ceddf3894d00b2488174830077f8, entries=150, sequenceid=40, filesize=11.7 K 2024-11-09T02:22:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119038380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:58,388 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a695692878ab8811a17602b58ad8f4dd in 1315ms, sequenceid=40, compaction requested=false 2024-11-09T02:22:58,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:22:58,388 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_0' in a695692878ab8811a17602b58ad8f4dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:22:58,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:22:58,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-09T02:22:58,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:58,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:58,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-09T02:22:58,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-09T02:22:58,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2410 sec 2024-11-09T02:22:58,393 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 2.2510 sec 2024-11-09T02:22:58,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:58,504 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:22:58,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a695692878ab8811a17602b58ad8f4dd 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T02:22:58,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f209c1d483444e2f9e35777c93a09068_a695692878ab8811a17602b58ad8f4dd is 50, key is test_row_0/A:col10/1731118977355/Put/seqid=0 2024-11-09T02:22:58,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742154_1330 (size=17034) 2024-11-09T02:22:58,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:58,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119038563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:58,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:58,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119038563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:58,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:58,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119038565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:58,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:58,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:58,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:58,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119038567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:58,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:58,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:58,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:58,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:58,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:58,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119038670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:58,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:58,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119038670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:58,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:58,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119038680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:58,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:58,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:58,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:58,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119038690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:58,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:58,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:58,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:58,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:58,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119038880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:58,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:58,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:58,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:58,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:58,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119038890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:58,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:58,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:58,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:58,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:58,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119038890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:58,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:58,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:58,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:58,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119038900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:58,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:58,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:58,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:58,923 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:22:58,927 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f209c1d483444e2f9e35777c93a09068_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f209c1d483444e2f9e35777c93a09068_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:22:58,928 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/6637d17800b94ec188bbad4822c5dbc9, store: [table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:22:58,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/6637d17800b94ec188bbad4822c5dbc9 is 175, key is test_row_0/A:col10/1731118977355/Put/seqid=0 2024-11-09T02:22:58,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742155_1331 (size=48139) 2024-11-09T02:22:59,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:59,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119039190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:59,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:59,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:59,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:59,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:59,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119039200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:59,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:59,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:59,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:59,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119039200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:59,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:59,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:59,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:59,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:59,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119039210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:59,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:59,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:59,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:59,342 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/6637d17800b94ec188bbad4822c5dbc9 2024-11-09T02:22:59,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/12ba5e23c2cc4669856c78a4ad4f48a3 is 50, key is test_row_0/B:col10/1731118977355/Put/seqid=0 2024-11-09T02:22:59,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742156_1332 (size=12001) 2024-11-09T02:22:59,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:59,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119039700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:59,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:59,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:59,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:59,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:59,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119039710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:59,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:59,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:59,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:59,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119039720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:59,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:22:59,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119039729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:22:59,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:59,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:22:59,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:22:59,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/12ba5e23c2cc4669856c78a4ad4f48a3 2024-11-09T02:22:59,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/ca9ca600fd274c2a8451d752187c247e is 50, key is test_row_0/C:col10/1731118977355/Put/seqid=0 2024-11-09T02:22:59,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742157_1333 (size=12001) 2024-11-09T02:23:00,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/ca9ca600fd274c2a8451d752187c247e 2024-11-09T02:23:00,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/6637d17800b94ec188bbad4822c5dbc9 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/6637d17800b94ec188bbad4822c5dbc9 2024-11-09T02:23:00,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/6637d17800b94ec188bbad4822c5dbc9, entries=250, sequenceid=53, filesize=47.0 K 2024-11-09T02:23:00,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/12ba5e23c2cc4669856c78a4ad4f48a3 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/12ba5e23c2cc4669856c78a4ad4f48a3 2024-11-09T02:23:00,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/12ba5e23c2cc4669856c78a4ad4f48a3, entries=150, sequenceid=53, filesize=11.7 K 2024-11-09T02:23:00,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/ca9ca600fd274c2a8451d752187c247e as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/ca9ca600fd274c2a8451d752187c247e 2024-11-09T02:23:00,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/ca9ca600fd274c2a8451d752187c247e, entries=150, sequenceid=53, filesize=11.7 K 2024-11-09T02:23:00,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for a695692878ab8811a17602b58ad8f4dd in 1677ms, sequenceid=53, compaction requested=true 2024-11-09T02:23:00,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:00,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a695692878ab8811a17602b58ad8f4dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:23:00,182 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:00,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:00,182 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:00,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a695692878ab8811a17602b58ad8f4dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:23:00,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:00,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a695692878ab8811a17602b58ad8f4dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:23:00,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:00,183 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:00,183 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): a695692878ab8811a17602b58ad8f4dd/A is initiating minor compaction (all files) 2024-11-09T02:23:00,183 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a695692878ab8811a17602b58ad8f4dd/A in TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:00,183 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/74b4ce6bbde54972af252da329afa084, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/8a6d0d322bdb495f9c6732027dbf6e03, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/6637d17800b94ec188bbad4822c5dbc9] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp, totalSize=115.9 K 2024-11-09T02:23:00,183 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:00,183 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. files: [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/74b4ce6bbde54972af252da329afa084, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/8a6d0d322bdb495f9c6732027dbf6e03, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/6637d17800b94ec188bbad4822c5dbc9] 2024-11-09T02:23:00,183 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:00,183 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): a695692878ab8811a17602b58ad8f4dd/B is initiating minor compaction (all files) 2024-11-09T02:23:00,183 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a695692878ab8811a17602b58ad8f4dd/B in TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:00,183 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/b24c730f62a643b5a9c23acd9ba14919, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/7b6a921ee6a840d186a8a30a8e280041, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/12ba5e23c2cc4669856c78a4ad4f48a3] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp, totalSize=35.2 K 2024-11-09T02:23:00,183 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74b4ce6bbde54972af252da329afa084, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731118976148 2024-11-09T02:23:00,184 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting b24c730f62a643b5a9c23acd9ba14919, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731118976148 2024-11-09T02:23:00,184 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a6d0d322bdb495f9c6732027dbf6e03, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731118976187 2024-11-09T02:23:00,184 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b6a921ee6a840d186a8a30a8e280041, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731118976187 2024-11-09T02:23:00,184 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6637d17800b94ec188bbad4822c5dbc9, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731118977350 2024-11-09T02:23:00,184 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 12ba5e23c2cc4669856c78a4ad4f48a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731118977355 2024-11-09T02:23:00,191 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a695692878ab8811a17602b58ad8f4dd#B#compaction#282 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:00,191 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/27e420f2e3e6416ebd57102df880a826 is 50, key is test_row_0/B:col10/1731118977355/Put/seqid=0 2024-11-09T02:23:00,196 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:00,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742158_1334 (size=12104) 2024-11-09T02:23:00,200 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411090152e474e203482c9d269b936bd81923_a695692878ab8811a17602b58ad8f4dd store=[table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:00,202 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411090152e474e203482c9d269b936bd81923_a695692878ab8811a17602b58ad8f4dd, store=[table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:00,202 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090152e474e203482c9d269b936bd81923_a695692878ab8811a17602b58ad8f4dd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:00,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742159_1335 (size=4469) 2024-11-09T02:23:00,207 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/27e420f2e3e6416ebd57102df880a826 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/27e420f2e3e6416ebd57102df880a826 2024-11-09T02:23:00,211 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a695692878ab8811a17602b58ad8f4dd/B of a695692878ab8811a17602b58ad8f4dd into 27e420f2e3e6416ebd57102df880a826(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:00,211 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:00,211 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., storeName=a695692878ab8811a17602b58ad8f4dd/B, priority=13, startTime=1731118980182; duration=0sec 2024-11-09T02:23:00,211 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:00,211 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a695692878ab8811a17602b58ad8f4dd:B 2024-11-09T02:23:00,211 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:00,212 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:00,212 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): a695692878ab8811a17602b58ad8f4dd/C is initiating minor compaction (all files) 2024-11-09T02:23:00,212 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a695692878ab8811a17602b58ad8f4dd/C in TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:00,212 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/9e14c1cfa9f04886a6525830711da1e8, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/57c2ceddf3894d00b2488174830077f8, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/ca9ca600fd274c2a8451d752187c247e] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp, totalSize=35.2 K 2024-11-09T02:23:00,212 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e14c1cfa9f04886a6525830711da1e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731118976148 2024-11-09T02:23:00,212 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 57c2ceddf3894d00b2488174830077f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731118976187 2024-11-09T02:23:00,213 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting ca9ca600fd274c2a8451d752187c247e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731118977355 2024-11-09T02:23:00,220 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a695692878ab8811a17602b58ad8f4dd#C#compaction#284 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:00,220 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/4f08e3604f8143829f6f025b75714715 is 50, key is test_row_0/C:col10/1731118977355/Put/seqid=0 2024-11-09T02:23:00,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742160_1336 (size=12104) 2024-11-09T02:23:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-09T02:23:00,310 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:23:00,310 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:23:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-09T02:23:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-09T02:23:00,311 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:23:00,312 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:23:00,312 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:23:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:00,418 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:23:00,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a695692878ab8811a17602b58ad8f4dd 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-09T02:23:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-09T02:23:00,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095c7584c2b612468287294d8c5036f870_a695692878ab8811a17602b58ad8f4dd is 50, key is test_row_0/A:col10/1731118980417/Put/seqid=0 2024-11-09T02:23:00,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742161_1337 (size=12154) 2024-11-09T02:23:00,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-09T02:23:00,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:00,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:00,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:00,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:00,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:00,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119040476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:00,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:00,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:00,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:00,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119040590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:00,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:00,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:00,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:00,609 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a695692878ab8811a17602b58ad8f4dd#A#compaction#283 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:00,609 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/3502cb78c75f45b388f75d621e193036 is 175, key is test_row_0/A:col10/1731118977355/Put/seqid=0 2024-11-09T02:23:00,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742162_1338 (size=31058) 2024-11-09T02:23:00,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-09T02:23:00,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:00,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:00,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:00,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:00,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-09T02:23:00,635 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/4f08e3604f8143829f6f025b75714715 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/4f08e3604f8143829f6f025b75714715 2024-11-09T02:23:00,641 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a695692878ab8811a17602b58ad8f4dd/C of a695692878ab8811a17602b58ad8f4dd into 4f08e3604f8143829f6f025b75714715(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:00,641 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:00,641 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., storeName=a695692878ab8811a17602b58ad8f4dd/C, priority=13, startTime=1731118980182; duration=0sec 2024-11-09T02:23:00,641 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:00,641 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a695692878ab8811a17602b58ad8f4dd:C 2024-11-09T02:23:00,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119040710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:00,720 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_1' in a695692878ab8811a17602b58ad8f4dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:23:00,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:00,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:00,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:00,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119040719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:00,724 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_0' in a695692878ab8811a17602b58ad8f4dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:23:00,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:00,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:00,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:00,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119040740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:00,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119040740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:00,749 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_0' in a695692878ab8811a17602b58ad8f4dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:23:00,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:00,749 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_1' in a695692878ab8811a17602b58ad8f4dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:23:00,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:00,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:00,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:00,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:00,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:00,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-09T02:23:00,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:00,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:00,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:00,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:00,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:00,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:00,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119040799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:00,827 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:00,830 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095c7584c2b612468287294d8c5036f870_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095c7584c2b612468287294d8c5036f870_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:00,830 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/6a65bd5aff4b402880436f5744d15ea6, store: [table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:00,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/6a65bd5aff4b402880436f5744d15ea6 is 175, key is test_row_0/A:col10/1731118980417/Put/seqid=0 2024-11-09T02:23:00,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742163_1339 (size=30955) 2024-11-09T02:23:00,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-09T02:23:00,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:00,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:00,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:00,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:00,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:00,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:00,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-09T02:23:01,018 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/3502cb78c75f45b388f75d621e193036 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/3502cb78c75f45b388f75d621e193036 2024-11-09T02:23:01,022 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a695692878ab8811a17602b58ad8f4dd/A of a695692878ab8811a17602b58ad8f4dd into 3502cb78c75f45b388f75d621e193036(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:01,022 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:01,022 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., storeName=a695692878ab8811a17602b58ad8f4dd/A, priority=13, startTime=1731118980182; duration=0sec 2024-11-09T02:23:01,022 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:01,022 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a695692878ab8811a17602b58ad8f4dd:A 2024-11-09T02:23:01,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-09T02:23:01,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:01,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:01,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:01,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:01,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:01,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:01,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119041110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:01,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-09T02:23:01,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:01,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:01,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:01,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:01,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:01,235 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/6a65bd5aff4b402880436f5744d15ea6 2024-11-09T02:23:01,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/6118d4a29a044ae389a43992c14cc198 is 50, key is test_row_0/B:col10/1731118980417/Put/seqid=0 2024-11-09T02:23:01,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742164_1340 (size=12001) 2024-11-09T02:23:01,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-09T02:23:01,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:01,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:01,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:01,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:01,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:01,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-09T02:23:01,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-09T02:23:01,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:01,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:01,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:01,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:01,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:01,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:01,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119041620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:01,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/6118d4a29a044ae389a43992c14cc198 2024-11-09T02:23:01,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/0f103651fbd84b97ab26ccf8f1758f13 is 50, key is test_row_0/C:col10/1731118980417/Put/seqid=0 2024-11-09T02:23:01,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742165_1341 (size=12001) 2024-11-09T02:23:01,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/0f103651fbd84b97ab26ccf8f1758f13 2024-11-09T02:23:01,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/6a65bd5aff4b402880436f5744d15ea6 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/6a65bd5aff4b402880436f5744d15ea6 2024-11-09T02:23:01,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/6a65bd5aff4b402880436f5744d15ea6, entries=150, sequenceid=78, filesize=30.2 K 2024-11-09T02:23:01,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/6118d4a29a044ae389a43992c14cc198 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/6118d4a29a044ae389a43992c14cc198 2024-11-09T02:23:01,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/6118d4a29a044ae389a43992c14cc198, entries=150, sequenceid=78, filesize=11.7 K 2024-11-09T02:23:01,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/0f103651fbd84b97ab26ccf8f1758f13 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/0f103651fbd84b97ab26ccf8f1758f13 2024-11-09T02:23:01,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/0f103651fbd84b97ab26ccf8f1758f13, entries=150, sequenceid=78, filesize=11.7 K 2024-11-09T02:23:01,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for a695692878ab8811a17602b58ad8f4dd in 1255ms, sequenceid=78, compaction requested=false 2024-11-09T02:23:01,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:01,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-09T02:23:01,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:01,686 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing a695692878ab8811a17602b58ad8f4dd 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T02:23:01,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109926bf121189f448e97b4afe727f5c9c7_a695692878ab8811a17602b58ad8f4dd is 50, key is test_row_0/A:col10/1731118980460/Put/seqid=0 2024-11-09T02:23:01,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742166_1342 (size=12154) 2024-11-09T02:23:02,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,100 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109926bf121189f448e97b4afe727f5c9c7_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109926bf121189f448e97b4afe727f5c9c7_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:02,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/3b0c724f3746434bb229e0c3e3e099d3, store: [table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:02,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/3b0c724f3746434bb229e0c3e3e099d3 is 175, key is test_row_0/A:col10/1731118980460/Put/seqid=0 2024-11-09T02:23:02,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742167_1343 (size=30955) 2024-11-09T02:23:02,112 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/3b0c724f3746434bb229e0c3e3e099d3 2024-11-09T02:23:02,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/5c006ffcdbb547a5b9458b70114dc908 is 50, key is test_row_0/B:col10/1731118980460/Put/seqid=0 2024-11-09T02:23:02,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742168_1344 (size=12001) 2024-11-09T02:23:02,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-09T02:23:02,524 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/5c006ffcdbb547a5b9458b70114dc908 2024-11-09T02:23:02,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/fb2c76960dfc4eb6a3dafca15c11df27 is 50, key is test_row_0/C:col10/1731118980460/Put/seqid=0 2024-11-09T02:23:02,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742169_1345 (size=12001) 2024-11-09T02:23:02,535 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/fb2c76960dfc4eb6a3dafca15c11df27 2024-11-09T02:23:02,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/3b0c724f3746434bb229e0c3e3e099d3 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/3b0c724f3746434bb229e0c3e3e099d3 2024-11-09T02:23:02,545 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/3b0c724f3746434bb229e0c3e3e099d3, entries=150, sequenceid=92, filesize=30.2 K 2024-11-09T02:23:02,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/5c006ffcdbb547a5b9458b70114dc908 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/5c006ffcdbb547a5b9458b70114dc908 2024-11-09T02:23:02,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,549 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/5c006ffcdbb547a5b9458b70114dc908, entries=150, sequenceid=92, filesize=11.7 K 2024-11-09T02:23:02,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/fb2c76960dfc4eb6a3dafca15c11df27 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/fb2c76960dfc4eb6a3dafca15c11df27 2024-11-09T02:23:02,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,553 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/fb2c76960dfc4eb6a3dafca15c11df27, entries=150, sequenceid=92, filesize=11.7 K 2024-11-09T02:23:02,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,554 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for a695692878ab8811a17602b58ad8f4dd in 868ms, sequenceid=92, compaction requested=true 2024-11-09T02:23:02,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:02,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:02,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-09T02:23:02,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-09T02:23:02,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-09T02:23:02,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2430 sec 2024-11-09T02:23:02,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,559 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.2470 sec 2024-11-09T02:23:02,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:02,824 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:23:02,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a695692878ab8811a17602b58ad8f4dd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T02:23:02,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110933b056a948f4412b90a9a10092109f4f_a695692878ab8811a17602b58ad8f4dd is 50, key is test_row_0/A:col10/1731118982816/Put/seqid=0 2024-11-09T02:23:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742170_1346 (size=17034) 2024-11-09T02:23:02,876 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,880 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110933b056a948f4412b90a9a10092109f4f_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110933b056a948f4412b90a9a10092109f4f_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:02,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,882 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/13995700c51c4b7e8592c3770357de05, store: [table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:02,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:02,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/13995700c51c4b7e8592c3770357de05 is 175, key is test_row_0/A:col10/1731118982816/Put/seqid=0 2024-11-09T02:23:02,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742171_1347 (size=48139) 2024-11-09T02:23:02,902 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=103, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/13995700c51c4b7e8592c3770357de05 2024-11-09T02:23:02,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/883a7493a79043a9a484506c78cba62b is 50, key is test_row_0/B:col10/1731118982816/Put/seqid=0 2024-11-09T02:23:02,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742172_1348 (size=12001) 2024-11-09T02:23:02,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119042910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:02,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:02,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:02,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:02,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119042911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:02,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:02,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:02,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119042912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:02,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:02,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:02,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:02,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:02,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119042914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:02,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:02,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:02,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:02,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119042916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:02,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:02,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:02,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:03,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119043040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:03,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:03,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119043040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:03,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:03,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:03,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119043040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:03,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:03,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:03,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119043040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:03,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:03,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119043040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:03,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:03,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119043260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:03,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:03,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119043260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:03,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:03,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:03,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119043260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:03,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:03,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:03,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119043260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:03,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:03,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:03,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119043260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:03,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:03,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/883a7493a79043a9a484506c78cba62b 2024-11-09T02:23:03,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/626079bbd380479383f90cc91f625394 is 50, key is test_row_0/C:col10/1731118982816/Put/seqid=0 2024-11-09T02:23:03,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742173_1349 (size=12001) 2024-11-09T02:23:03,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/626079bbd380479383f90cc91f625394 2024-11-09T02:23:03,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/13995700c51c4b7e8592c3770357de05 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/13995700c51c4b7e8592c3770357de05 2024-11-09T02:23:03,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/13995700c51c4b7e8592c3770357de05, entries=250, sequenceid=103, filesize=47.0 K 2024-11-09T02:23:03,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/883a7493a79043a9a484506c78cba62b as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/883a7493a79043a9a484506c78cba62b 2024-11-09T02:23:03,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/883a7493a79043a9a484506c78cba62b, entries=150, sequenceid=103, filesize=11.7 K 2024-11-09T02:23:03,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/626079bbd380479383f90cc91f625394 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/626079bbd380479383f90cc91f625394 2024-11-09T02:23:03,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/626079bbd380479383f90cc91f625394, entries=150, sequenceid=103, filesize=11.7 K 2024-11-09T02:23:03,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a695692878ab8811a17602b58ad8f4dd in 585ms, sequenceid=103, compaction requested=true 2024-11-09T02:23:03,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:03,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a695692878ab8811a17602b58ad8f4dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:23:03,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:03,409 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T02:23:03,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a695692878ab8811a17602b58ad8f4dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:23:03,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:03,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a695692878ab8811a17602b58ad8f4dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:23:03,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T02:23:03,410 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T02:23:03,410 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T02:23:03,410 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): a695692878ab8811a17602b58ad8f4dd/A is initiating minor compaction (all files) 2024-11-09T02:23:03,410 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a695692878ab8811a17602b58ad8f4dd/A in TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:03,411 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/3502cb78c75f45b388f75d621e193036, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/6a65bd5aff4b402880436f5744d15ea6, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/3b0c724f3746434bb229e0c3e3e099d3, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/13995700c51c4b7e8592c3770357de05] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp, totalSize=137.8 K 2024-11-09T02:23:03,411 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T02:23:03,411 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): a695692878ab8811a17602b58ad8f4dd/B is initiating minor compaction (all files) 2024-11-09T02:23:03,411 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:03,411 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a695692878ab8811a17602b58ad8f4dd/B in TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:03,411 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. files: [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/3502cb78c75f45b388f75d621e193036, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/6a65bd5aff4b402880436f5744d15ea6, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/3b0c724f3746434bb229e0c3e3e099d3, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/13995700c51c4b7e8592c3770357de05] 2024-11-09T02:23:03,411 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/27e420f2e3e6416ebd57102df880a826, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/6118d4a29a044ae389a43992c14cc198, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/5c006ffcdbb547a5b9458b70114dc908, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/883a7493a79043a9a484506c78cba62b] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp, totalSize=47.0 K 2024-11-09T02:23:03,411 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3502cb78c75f45b388f75d621e193036, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731118977355 2024-11-09T02:23:03,411 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 27e420f2e3e6416ebd57102df880a826, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731118977355 2024-11-09T02:23:03,412 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 6118d4a29a044ae389a43992c14cc198, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731118978553 2024-11-09T02:23:03,412 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a65bd5aff4b402880436f5744d15ea6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731118978553 2024-11-09T02:23:03,412 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c006ffcdbb547a5b9458b70114dc908, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731118980460 2024-11-09T02:23:03,412 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b0c724f3746434bb229e0c3e3e099d3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731118980460 2024-11-09T02:23:03,412 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 883a7493a79043a9a484506c78cba62b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731118982816 2024-11-09T02:23:03,412 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13995700c51c4b7e8592c3770357de05, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731118982783 2024-11-09T02:23:03,421 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:03,421 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a695692878ab8811a17602b58ad8f4dd#B#compaction#294 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:03,422 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/0f8b2c0fe4424c96946161deddec14b5 is 50, key is test_row_0/B:col10/1731118982816/Put/seqid=0 2024-11-09T02:23:03,429 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109616e3b66d0a440329a5a26efdb3a2252_a695692878ab8811a17602b58ad8f4dd store=[table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:03,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742174_1350 (size=12241) 2024-11-09T02:23:03,432 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109616e3b66d0a440329a5a26efdb3a2252_a695692878ab8811a17602b58ad8f4dd, store=[table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:03,432 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109616e3b66d0a440329a5a26efdb3a2252_a695692878ab8811a17602b58ad8f4dd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:03,438 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/0f8b2c0fe4424c96946161deddec14b5 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/0f8b2c0fe4424c96946161deddec14b5 2024-11-09T02:23:03,445 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a695692878ab8811a17602b58ad8f4dd/B of a695692878ab8811a17602b58ad8f4dd into 0f8b2c0fe4424c96946161deddec14b5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:03,445 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:03,445 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., storeName=a695692878ab8811a17602b58ad8f4dd/B, priority=12, startTime=1731118983409; duration=0sec 2024-11-09T02:23:03,445 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:03,445 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a695692878ab8811a17602b58ad8f4dd:B 2024-11-09T02:23:03,445 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T02:23:03,446 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T02:23:03,446 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): a695692878ab8811a17602b58ad8f4dd/C is initiating minor compaction (all files) 2024-11-09T02:23:03,446 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a695692878ab8811a17602b58ad8f4dd/C in TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:03,446 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/4f08e3604f8143829f6f025b75714715, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/0f103651fbd84b97ab26ccf8f1758f13, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/fb2c76960dfc4eb6a3dafca15c11df27, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/626079bbd380479383f90cc91f625394] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp, totalSize=47.0 K 2024-11-09T02:23:03,447 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f08e3604f8143829f6f025b75714715, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731118977355 2024-11-09T02:23:03,447 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f103651fbd84b97ab26ccf8f1758f13, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731118978553 2024-11-09T02:23:03,447 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting fb2c76960dfc4eb6a3dafca15c11df27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731118980460 2024-11-09T02:23:03,447 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 626079bbd380479383f90cc91f625394, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731118982816 2024-11-09T02:23:03,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742175_1351 (size=4469) 2024-11-09T02:23:03,458 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a695692878ab8811a17602b58ad8f4dd#C#compaction#296 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:03,459 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/70650045a6a149919360170d66ae4e68 is 50, key is test_row_0/C:col10/1731118982816/Put/seqid=0 2024-11-09T02:23:03,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742176_1352 (size=12241) 2024-11-09T02:23:03,475 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/70650045a6a149919360170d66ae4e68 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/70650045a6a149919360170d66ae4e68 2024-11-09T02:23:03,479 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a695692878ab8811a17602b58ad8f4dd/C of a695692878ab8811a17602b58ad8f4dd into 70650045a6a149919360170d66ae4e68(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:03,479 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:03,479 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., storeName=a695692878ab8811a17602b58ad8f4dd/C, priority=12, startTime=1731118983409; duration=0sec 2024-11-09T02:23:03,479 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:03,479 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a695692878ab8811a17602b58ad8f4dd:C 2024-11-09T02:23:03,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:03,573 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:23:03,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a695692878ab8811a17602b58ad8f4dd 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-09T02:23:03,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411092e9de61d81f44ad4bd7a62d3de377cc4_a695692878ab8811a17602b58ad8f4dd is 50, key is test_row_0/A:col10/1731118982913/Put/seqid=0 2024-11-09T02:23:03,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119043580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:03,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:03,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:03,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119043584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:03,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:03,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742177_1353 (size=17134) 2024-11-09T02:23:03,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:03,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119043585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:03,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:03,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119043585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:03,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:03,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119043586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:03,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:03,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:03,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119043690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:03,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:03,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119043700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:03,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:03,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119043700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:03,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:03,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:03,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119043700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:03,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:03,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:03,854 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a695692878ab8811a17602b58ad8f4dd#A#compaction#295 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:03,854 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/72fd3892aa5d4e5aa274353f6811a36f is 175, key is test_row_0/A:col10/1731118982816/Put/seqid=0 2024-11-09T02:23:03,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742178_1354 (size=31195) 2024-11-09T02:23:03,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:03,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119043900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:03,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:03,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119043910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119043910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:03,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119043910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:03,988 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:03,991 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411092e9de61d81f44ad4bd7a62d3de377cc4_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092e9de61d81f44ad4bd7a62d3de377cc4_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:03,992 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/129b7d4c2ee240889da3e9861e4a9412, store: [table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:03,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/129b7d4c2ee240889da3e9861e4a9412 is 175, key is test_row_0/A:col10/1731118982913/Put/seqid=0 2024-11-09T02:23:03,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742179_1355 (size=48239) 2024-11-09T02:23:03,996 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/129b7d4c2ee240889da3e9861e4a9412 2024-11-09T02:23:04,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/e6d06ee3b3b9479a9edfeba04d4768ea is 50, key is test_row_0/B:col10/1731118982913/Put/seqid=0 2024-11-09T02:23:04,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742180_1356 (size=12051) 2024-11-09T02:23:04,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/e6d06ee3b3b9479a9edfeba04d4768ea 2024-11-09T02:23:04,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/81035b55be6b4968b458f1ed0cc503ba is 50, key is test_row_0/C:col10/1731118982913/Put/seqid=0 2024-11-09T02:23:04,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742181_1357 (size=12051) 2024-11-09T02:23:04,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:04,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119044100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:04,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:04,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:04,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:04,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:04,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119044209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:04,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:04,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:04,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:04,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:04,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119044220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:04,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:04,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119044220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:04,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:04,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119044220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:04,262 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/72fd3892aa5d4e5aa274353f6811a36f as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/72fd3892aa5d4e5aa274353f6811a36f 2024-11-09T02:23:04,265 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a695692878ab8811a17602b58ad8f4dd/A of a695692878ab8811a17602b58ad8f4dd into 72fd3892aa5d4e5aa274353f6811a36f(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:04,265 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:04,265 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., storeName=a695692878ab8811a17602b58ad8f4dd/A, priority=12, startTime=1731118983409; duration=0sec 2024-11-09T02:23:04,265 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:04,265 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a695692878ab8811a17602b58ad8f4dd:A 2024-11-09T02:23:04,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/81035b55be6b4968b458f1ed0cc503ba 2024-11-09T02:23:04,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/129b7d4c2ee240889da3e9861e4a9412 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/129b7d4c2ee240889da3e9861e4a9412 2024-11-09T02:23:04,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/129b7d4c2ee240889da3e9861e4a9412, entries=250, sequenceid=132, filesize=47.1 K 2024-11-09T02:23:04,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/e6d06ee3b3b9479a9edfeba04d4768ea as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/e6d06ee3b3b9479a9edfeba04d4768ea 2024-11-09T02:23:04,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/e6d06ee3b3b9479a9edfeba04d4768ea, entries=150, sequenceid=132, filesize=11.8 K 2024-11-09T02:23:04,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/81035b55be6b4968b458f1ed0cc503ba as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/81035b55be6b4968b458f1ed0cc503ba 2024-11-09T02:23:04,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/81035b55be6b4968b458f1ed0cc503ba, entries=150, sequenceid=132, filesize=11.8 K 2024-11-09T02:23:04,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for a695692878ab8811a17602b58ad8f4dd in 860ms, sequenceid=132, compaction requested=false 2024-11-09T02:23:04,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:04,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-09T02:23:04,480 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:23:04,480 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:23:04,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-09T02:23:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-09T02:23:04,484 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:23:04,485 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:23:04,485 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:23:04,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-09T02:23:04,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-09T02:23:04,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:04,637 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing a695692878ab8811a17602b58ad8f4dd 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-09T02:23:04,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d0dfbbbec9284eb4928546873883be7b_a695692878ab8811a17602b58ad8f4dd is 50, key is test_row_0/A:col10/1731118983584/Put/seqid=0 2024-11-09T02:23:04,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742182_1358 (size=12304) 2024-11-09T02:23:04,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:04,650 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d0dfbbbec9284eb4928546873883be7b_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d0dfbbbec9284eb4928546873883be7b_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:04,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/ba62ea1f1a9d450988703e8e08ce7ce6, store: [table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:04,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/ba62ea1f1a9d450988703e8e08ce7ce6 is 175, key is test_row_0/A:col10/1731118983584/Put/seqid=0 2024-11-09T02:23:04,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742183_1359 (size=31105) 2024-11-09T02:23:04,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:04,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:04,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:04,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119044788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:04,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:04,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119044788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:04,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:04,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:04,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:04,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:04,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:04,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:04,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:04,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119044789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:04,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:04,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:04,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:04,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:04,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119044790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:04,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:04,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:04,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:04,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-09T02:23:04,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:04,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119044900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:04,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:04,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119044900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:04,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:04,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119044900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:04,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:04,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119044910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:04,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:04,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:04,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:05,055 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=142, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/ba62ea1f1a9d450988703e8e08ce7ce6 2024-11-09T02:23:05,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/ff9a5320d7074362a803d3f526b64c4e is 50, key is test_row_0/B:col10/1731118983584/Put/seqid=0 2024-11-09T02:23:05,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742184_1360 (size=12151) 2024-11-09T02:23:05,064 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/ff9a5320d7074362a803d3f526b64c4e 2024-11-09T02:23:05,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/ad8295af51d54539932586a5e9ec5bbd is 50, key is test_row_0/C:col10/1731118983584/Put/seqid=0 2024-11-09T02:23:05,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742185_1361 (size=12151) 2024-11-09T02:23:05,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-09T02:23:05,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119045110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:05,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:05,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:05,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:05,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119045120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:05,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119045120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:05,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119045120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:05,126 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_0' in a695692878ab8811a17602b58ad8f4dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:23:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:05,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119045120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:05,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:05,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:05,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:05,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:05,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119045419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:05,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119045430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:05,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119045430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:05,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:05,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:05,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:05,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:05,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119045440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:05,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:05,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:05,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:05,473 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/ad8295af51d54539932586a5e9ec5bbd 2024-11-09T02:23:05,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/ba62ea1f1a9d450988703e8e08ce7ce6 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/ba62ea1f1a9d450988703e8e08ce7ce6 2024-11-09T02:23:05,481 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/ba62ea1f1a9d450988703e8e08ce7ce6, entries=150, sequenceid=142, filesize=30.4 K 2024-11-09T02:23:05,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/ff9a5320d7074362a803d3f526b64c4e as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/ff9a5320d7074362a803d3f526b64c4e 2024-11-09T02:23:05,486 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/ff9a5320d7074362a803d3f526b64c4e, entries=150, sequenceid=142, filesize=11.9 K 2024-11-09T02:23:05,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/ad8295af51d54539932586a5e9ec5bbd as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/ad8295af51d54539932586a5e9ec5bbd 2024-11-09T02:23:05,491 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/ad8295af51d54539932586a5e9ec5bbd, entries=150, sequenceid=142, filesize=11.9 K 2024-11-09T02:23:05,491 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for a695692878ab8811a17602b58ad8f4dd in 854ms, sequenceid=142, compaction requested=true 2024-11-09T02:23:05,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:05,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:05,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-09T02:23:05,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-09T02:23:05,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-09T02:23:05,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0080 sec 2024-11-09T02:23:05,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.0150 sec 2024-11-09T02:23:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-09T02:23:05,620 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:23:05,620 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:23:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-09T02:23:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-09T02:23:05,622 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:23:05,622 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:23:05,623 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:23:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-09T02:23:05,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T02:23:05,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:05,775 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing a695692878ab8811a17602b58ad8f4dd 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-09T02:23:05,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109292a8b67ab5f4f57aac3dfa3d25a4069_a695692878ab8811a17602b58ad8f4dd is 50, key is test_row_0/A:col10/1731118984787/Put/seqid=0 2024-11-09T02:23:05,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742186_1362 (size=12304) 2024-11-09T02:23:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-09T02:23:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:05,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:05,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119045958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:05,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:05,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:05,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:05,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:05,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119045959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119045961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:05,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:05,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:05,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119045961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:05,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:05,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:05,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:05,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:05,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:05,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:05,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:06,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119046069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:06,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:06,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:06,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:06,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119046080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:06,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119046080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:06,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:06,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:06,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:06,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119046080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:06,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:06,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:06,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:06,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,189 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109292a8b67ab5f4f57aac3dfa3d25a4069_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109292a8b67ab5f4f57aac3dfa3d25a4069_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:06,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/744b66d8822e4323a72aba64888917ca, store: [table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:06,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/744b66d8822e4323a72aba64888917ca is 175, key is test_row_0/A:col10/1731118984787/Put/seqid=0 2024-11-09T02:23:06,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742187_1363 (size=31105) 2024-11-09T02:23:06,195 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/744b66d8822e4323a72aba64888917ca 2024-11-09T02:23:06,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/7c30b591713640f2b27ff576b908b27d is 50, key is test_row_0/B:col10/1731118984787/Put/seqid=0 2024-11-09T02:23:06,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742188_1364 (size=12151) 2024-11-09T02:23:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-09T02:23:06,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119046280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:06,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:06,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:06,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:06,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119046290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:06,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:06,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:06,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:06,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119046290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:06,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:06,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:06,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119046290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:06,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:06,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:06,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:06,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:06,607 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/7c30b591713640f2b27ff576b908b27d 2024-11-09T02:23:06,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119046600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:06,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119046600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:06,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119046600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:06,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:06,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:06,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:06,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119046600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:06,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:06,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:06,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:06,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/0c5e41d7034141f19ca68bde5013a259 is 50, key is test_row_0/C:col10/1731118984787/Put/seqid=0 2024-11-09T02:23:06,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742189_1365 (size=12151) 2024-11-09T02:23:06,616 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/0c5e41d7034141f19ca68bde5013a259 2024-11-09T02:23:06,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/744b66d8822e4323a72aba64888917ca as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/744b66d8822e4323a72aba64888917ca 2024-11-09T02:23:06,625 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/744b66d8822e4323a72aba64888917ca, entries=150, sequenceid=169, filesize=30.4 K 2024-11-09T02:23:06,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/7c30b591713640f2b27ff576b908b27d as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/7c30b591713640f2b27ff576b908b27d 2024-11-09T02:23:06,631 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/7c30b591713640f2b27ff576b908b27d, entries=150, sequenceid=169, filesize=11.9 K 2024-11-09T02:23:06,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/0c5e41d7034141f19ca68bde5013a259 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/0c5e41d7034141f19ca68bde5013a259 2024-11-09T02:23:06,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,636 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/0c5e41d7034141f19ca68bde5013a259, entries=150, sequenceid=169, filesize=11.9 K 2024-11-09T02:23:06,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,636 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for a695692878ab8811a17602b58ad8f4dd in 861ms, sequenceid=169, compaction requested=true 2024-11-09T02:23:06,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:06,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:06,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-09T02:23:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-09T02:23:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-09T02:23:06,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0160 sec 2024-11-09T02:23:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.0210 sec 2024-11-09T02:23:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-09T02:23:06,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,760 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:23:06,760 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:23:06,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-09T02:23:06,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-09T02:23:06,762 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:23:06,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,763 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:23:06,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,763 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:23:06,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-09T02:23:06,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-09T02:23:06,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:06,916 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing a695692878ab8811a17602b58ad8f4dd 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-09T02:23:06,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094132f07cbe4c4f889fa448a3f264c3e9_a695692878ab8811a17602b58ad8f4dd is 50, key is test_row_0/A:col10/1731118985950/Put/seqid=0 2024-11-09T02:23:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742190_1366 (size=9814) 2024-11-09T02:23:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,954 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094132f07cbe4c4f889fa448a3f264c3e9_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094132f07cbe4c4f889fa448a3f264c3e9_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/09e2ea63543248e099035bae98ed135b, store: [table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/09e2ea63543248e099035bae98ed135b is 175, key is test_row_0/A:col10/1731118985950/Put/seqid=0 2024-11-09T02:23:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742191_1367 (size=22461) 2024-11-09T02:23:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,967 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=178, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/09e2ea63543248e099035bae98ed135b 2024-11-09T02:23:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/1c09ac20fca443bea7b1b59729d9fd1c is 50, key is test_row_0/B:col10/1731118985950/Put/seqid=0 2024-11-09T02:23:06,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742192_1368 (size=9757) 2024-11-09T02:23:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,986 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/1c09ac20fca443bea7b1b59729d9fd1c 2024-11-09T02:23:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/855e59bef48643879202dedefe68dcdc is 50, key is test_row_0/C:col10/1731118985950/Put/seqid=0 2024-11-09T02:23:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742193_1369 (size=9757) 2024-11-09T02:23:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,016 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/855e59bef48643879202dedefe68dcdc 2024-11-09T02:23:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/09e2ea63543248e099035bae98ed135b as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/09e2ea63543248e099035bae98ed135b 2024-11-09T02:23:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,033 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/09e2ea63543248e099035bae98ed135b, entries=100, sequenceid=178, filesize=21.9 K 2024-11-09T02:23:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/1c09ac20fca443bea7b1b59729d9fd1c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/1c09ac20fca443bea7b1b59729d9fd1c 2024-11-09T02:23:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,039 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/1c09ac20fca443bea7b1b59729d9fd1c, entries=100, sequenceid=178, filesize=9.5 K 2024-11-09T02:23:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/855e59bef48643879202dedefe68dcdc as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/855e59bef48643879202dedefe68dcdc 2024-11-09T02:23:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,043 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/855e59bef48643879202dedefe68dcdc, entries=100, sequenceid=178, filesize=9.5 K 2024-11-09T02:23:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,044 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for a695692878ab8811a17602b58ad8f4dd in 128ms, sequenceid=178, compaction requested=true 2024-11-09T02:23:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:07,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:07,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-09T02:23:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-09T02:23:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,048 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-09T02:23:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,048 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 283 msec 2024-11-09T02:23:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,051 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 289 msec 2024-11-09T02:23:07,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-09T02:23:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,080 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:23:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,081 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:23:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-09T02:23:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,083 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:23:07,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T02:23:07,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,084 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:23:07,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:23:07,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T02:23:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:07,209 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:23:07,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a695692878ab8811a17602b58ad8f4dd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T02:23:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d50fb56eeae948d892774aed112bf079_a695692878ab8811a17602b58ad8f4dd is 50, key is test_row_0/A:col10/1731118987173/Put/seqid=0 2024-11-09T02:23:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T02:23:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:07,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:07,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:07,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:07,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742194_1370 (size=24758) 2024-11-09T02:23:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:07,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119047286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:07,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:07,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:07,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119047286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:07,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:07,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:07,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119047287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:07,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:07,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:07,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119047295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:07,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:07,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119047296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:07,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:07,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T02:23:07,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:07,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:07,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:07,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:07,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:07,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T02:23:07,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119047410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:07,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:07,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119047410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:07,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:07,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119047410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:07,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119047410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119047420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:07,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:07,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T02:23:07,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:07,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:07,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:07,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:07,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:07,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:07,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:07,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119047629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:07,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:07,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119047629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:07,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119047629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:07,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119047630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:07,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119047630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:07,652 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:07,655 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d50fb56eeae948d892774aed112bf079_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d50fb56eeae948d892774aed112bf079_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:07,656 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/56e9de76ad744c90b0037f77d0bff053, store: [table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:07,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/56e9de76ad744c90b0037f77d0bff053 is 175, key is test_row_0/A:col10/1731118987173/Put/seqid=0 2024-11-09T02:23:07,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742195_1371 (size=74394) 2024-11-09T02:23:07,660 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=189, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/56e9de76ad744c90b0037f77d0bff053 2024-11-09T02:23:07,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/3b9d5d1d70ce4696877a06844797a93a is 50, key is test_row_0/B:col10/1731118987173/Put/seqid=0 2024-11-09T02:23:07,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742196_1372 (size=12151) 2024-11-09T02:23:07,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T02:23:07,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:07,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:07,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:07,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:07,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:07,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:07,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T02:23:07,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T02:23:07,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:07,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:07,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:07,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:07,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:07,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:07,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:07,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119047950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:07,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119047950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:07,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119047950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:07,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119047950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:07,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119047950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:07,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:07,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:08,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T02:23:08,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:08,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:08,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:08,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:08,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:08,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:08,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/3b9d5d1d70ce4696877a06844797a93a 2024-11-09T02:23:08,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/a803131f19d540ac881fd362be5d8b1b is 50, key is test_row_0/C:col10/1731118987173/Put/seqid=0 2024-11-09T02:23:08,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742197_1373 (size=12151) 2024-11-09T02:23:08,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T02:23:08,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:08,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:08,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:08,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:08,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:08,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:08,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T02:23:08,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T02:23:08,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:08,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:08,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:08,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:08,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:08,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:08,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T02:23:08,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:08,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:08,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:08,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:08,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:08,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:08,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:08,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119048459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:08,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119048459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:08,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:08,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119048470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:08,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:08,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:08,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:08,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119048470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:08,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:08,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:08,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:08,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:08,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:08,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119048470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:08,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:08,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:08,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:08,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/a803131f19d540ac881fd362be5d8b1b 2024-11-09T02:23:08,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/56e9de76ad744c90b0037f77d0bff053 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/56e9de76ad744c90b0037f77d0bff053 2024-11-09T02:23:08,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/56e9de76ad744c90b0037f77d0bff053, entries=400, sequenceid=189, filesize=72.7 K 2024-11-09T02:23:08,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/3b9d5d1d70ce4696877a06844797a93a as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/3b9d5d1d70ce4696877a06844797a93a 2024-11-09T02:23:08,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/3b9d5d1d70ce4696877a06844797a93a, entries=150, sequenceid=189, filesize=11.9 K 2024-11-09T02:23:08,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/a803131f19d540ac881fd362be5d8b1b as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/a803131f19d540ac881fd362be5d8b1b 2024-11-09T02:23:08,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/a803131f19d540ac881fd362be5d8b1b, entries=150, sequenceid=189, filesize=11.9 K 2024-11-09T02:23:08,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a695692878ab8811a17602b58ad8f4dd in 1334ms, sequenceid=189, compaction requested=true 2024-11-09T02:23:08,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:08,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a695692878ab8811a17602b58ad8f4dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:23:08,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:08,543 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-09T02:23:08,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a695692878ab8811a17602b58ad8f4dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:23:08,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:08,543 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-09T02:23:08,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a695692878ab8811a17602b58ad8f4dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:23:08,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:08,545 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 70502 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-09T02:23:08,545 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 238499 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-09T02:23:08,545 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): a695692878ab8811a17602b58ad8f4dd/A is initiating minor compaction (all files) 2024-11-09T02:23:08,545 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): a695692878ab8811a17602b58ad8f4dd/B is initiating minor compaction (all files) 2024-11-09T02:23:08,545 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a695692878ab8811a17602b58ad8f4dd/B in TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:08,545 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a695692878ab8811a17602b58ad8f4dd/A in TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:08,545 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/72fd3892aa5d4e5aa274353f6811a36f, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/129b7d4c2ee240889da3e9861e4a9412, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/ba62ea1f1a9d450988703e8e08ce7ce6, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/744b66d8822e4323a72aba64888917ca, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/09e2ea63543248e099035bae98ed135b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/56e9de76ad744c90b0037f77d0bff053] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp, totalSize=232.9 K 2024-11-09T02:23:08,545 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/0f8b2c0fe4424c96946161deddec14b5, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/e6d06ee3b3b9479a9edfeba04d4768ea, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/ff9a5320d7074362a803d3f526b64c4e, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/7c30b591713640f2b27ff576b908b27d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/1c09ac20fca443bea7b1b59729d9fd1c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/3b9d5d1d70ce4696877a06844797a93a] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp, totalSize=68.8 K 2024-11-09T02:23:08,545 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:08,545 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. files: [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/72fd3892aa5d4e5aa274353f6811a36f, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/129b7d4c2ee240889da3e9861e4a9412, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/ba62ea1f1a9d450988703e8e08ce7ce6, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/744b66d8822e4323a72aba64888917ca, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/09e2ea63543248e099035bae98ed135b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/56e9de76ad744c90b0037f77d0bff053] 2024-11-09T02:23:08,546 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72fd3892aa5d4e5aa274353f6811a36f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731118982816 2024-11-09T02:23:08,546 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f8b2c0fe4424c96946161deddec14b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731118982816 2024-11-09T02:23:08,546 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting e6d06ee3b3b9479a9edfeba04d4768ea, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731118982908 2024-11-09T02:23:08,546 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 129b7d4c2ee240889da3e9861e4a9412, keycount=250, bloomtype=ROW, size=47.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731118982891 2024-11-09T02:23:08,547 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting ff9a5320d7074362a803d3f526b64c4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731118983583 2024-11-09T02:23:08,547 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba62ea1f1a9d450988703e8e08ce7ce6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731118983583 2024-11-09T02:23:08,547 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c30b591713640f2b27ff576b908b27d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731118984786 2024-11-09T02:23:08,547 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 744b66d8822e4323a72aba64888917ca, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731118984786 2024-11-09T02:23:08,547 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c09ac20fca443bea7b1b59729d9fd1c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731118985950 2024-11-09T02:23:08,547 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09e2ea63543248e099035bae98ed135b, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731118985950 2024-11-09T02:23:08,547 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b9d5d1d70ce4696877a06844797a93a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731118987173 2024-11-09T02:23:08,547 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56e9de76ad744c90b0037f77d0bff053, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731118987168 2024-11-09T02:23:08,558 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:08,560 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a695692878ab8811a17602b58ad8f4dd#B#compaction#312 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:08,560 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/09d9e7a8dae44725a30292d6a1b8289c is 50, key is test_row_0/B:col10/1731118987173/Put/seqid=0 2024-11-09T02:23:08,564 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110953d8f00765394ee2aa9b0ca4494489ba_a695692878ab8811a17602b58ad8f4dd store=[table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:08,569 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110953d8f00765394ee2aa9b0ca4494489ba_a695692878ab8811a17602b58ad8f4dd, store=[table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:08,570 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110953d8f00765394ee2aa9b0ca4494489ba_a695692878ab8811a17602b58ad8f4dd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:08,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742198_1374 (size=12595) 2024-11-09T02:23:08,582 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/09d9e7a8dae44725a30292d6a1b8289c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/09d9e7a8dae44725a30292d6a1b8289c 2024-11-09T02:23:08,588 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in a695692878ab8811a17602b58ad8f4dd/B of a695692878ab8811a17602b58ad8f4dd into 09d9e7a8dae44725a30292d6a1b8289c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:08,588 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:08,588 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., storeName=a695692878ab8811a17602b58ad8f4dd/B, priority=10, startTime=1731118988543; duration=0sec 2024-11-09T02:23:08,588 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:08,588 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a695692878ab8811a17602b58ad8f4dd:B 2024-11-09T02:23:08,588 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-09T02:23:08,590 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 70502 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-09T02:23:08,590 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): a695692878ab8811a17602b58ad8f4dd/C is initiating minor compaction (all files) 2024-11-09T02:23:08,590 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a695692878ab8811a17602b58ad8f4dd/C in TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:08,590 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/70650045a6a149919360170d66ae4e68, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/81035b55be6b4968b458f1ed0cc503ba, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/ad8295af51d54539932586a5e9ec5bbd, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/0c5e41d7034141f19ca68bde5013a259, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/855e59bef48643879202dedefe68dcdc, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/a803131f19d540ac881fd362be5d8b1b] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp, totalSize=68.8 K 2024-11-09T02:23:08,590 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 70650045a6a149919360170d66ae4e68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731118982816 2024-11-09T02:23:08,591 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 81035b55be6b4968b458f1ed0cc503ba, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731118982908 2024-11-09T02:23:08,591 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting ad8295af51d54539932586a5e9ec5bbd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731118983583 2024-11-09T02:23:08,591 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c5e41d7034141f19ca68bde5013a259, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731118984786 2024-11-09T02:23:08,591 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 855e59bef48643879202dedefe68dcdc, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731118985950 2024-11-09T02:23:08,592 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting a803131f19d540ac881fd362be5d8b1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731118987173 2024-11-09T02:23:08,607 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a695692878ab8811a17602b58ad8f4dd#C#compaction#314 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:08,608 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/6bbab525d2874879b86e4d8b53cd42b3 is 50, key is test_row_0/C:col10/1731118987173/Put/seqid=0 2024-11-09T02:23:08,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742199_1375 (size=4469) 2024-11-09T02:23:08,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T02:23:08,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:08,613 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a695692878ab8811a17602b58ad8f4dd#A#compaction#313 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:08,613 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing a695692878ab8811a17602b58ad8f4dd 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T02:23:08,614 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/4ae3063f9da248afb37e294cd4da3b5d is 175, key is test_row_0/A:col10/1731118987173/Put/seqid=0 2024-11-09T02:23:08,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742201_1377 (size=31549) 2024-11-09T02:23:08,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742200_1376 (size=12595) 2024-11-09T02:23:08,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099fc42cee334445c3b87ef0bc7737dca3_a695692878ab8811a17602b58ad8f4dd is 50, key is test_row_0/A:col10/1731118987286/Put/seqid=0 2024-11-09T02:23:08,640 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/4ae3063f9da248afb37e294cd4da3b5d as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/4ae3063f9da248afb37e294cd4da3b5d 2024-11-09T02:23:08,644 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in a695692878ab8811a17602b58ad8f4dd/A of a695692878ab8811a17602b58ad8f4dd into 4ae3063f9da248afb37e294cd4da3b5d(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:08,644 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:08,644 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., storeName=a695692878ab8811a17602b58ad8f4dd/A, priority=10, startTime=1731118988543; duration=0sec 2024-11-09T02:23:08,644 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:08,644 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a695692878ab8811a17602b58ad8f4dd:A 2024-11-09T02:23:08,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742202_1378 (size=12304) 2024-11-09T02:23:08,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:08,664 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099fc42cee334445c3b87ef0bc7737dca3_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099fc42cee334445c3b87ef0bc7737dca3_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:08,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/09ca3933ac7f4e8e89a189295d1b36e4, store: [table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:08,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/09ca3933ac7f4e8e89a189295d1b36e4 is 175, key is test_row_0/A:col10/1731118987286/Put/seqid=0 2024-11-09T02:23:08,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742203_1379 (size=31105) 2024-11-09T02:23:08,694 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/09ca3933ac7f4e8e89a189295d1b36e4 2024-11-09T02:23:08,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/76a07bbb007242599d2489a94d6a48e5 is 50, key is test_row_0/B:col10/1731118987286/Put/seqid=0 2024-11-09T02:23:08,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742204_1380 (size=12151) 2024-11-09T02:23:09,033 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/6bbab525d2874879b86e4d8b53cd42b3 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/6bbab525d2874879b86e4d8b53cd42b3 2024-11-09T02:23:09,037 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in a695692878ab8811a17602b58ad8f4dd/C of a695692878ab8811a17602b58ad8f4dd into 6bbab525d2874879b86e4d8b53cd42b3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:09,037 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:09,037 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., storeName=a695692878ab8811a17602b58ad8f4dd/C, priority=10, startTime=1731118988543; duration=0sec 2024-11-09T02:23:09,037 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:09,037 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a695692878ab8811a17602b58ad8f4dd:C 2024-11-09T02:23:09,120 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/76a07bbb007242599d2489a94d6a48e5 2024-11-09T02:23:09,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/a583e1e39b984d0eb4173c9b644a0db2 is 50, key is test_row_0/C:col10/1731118987286/Put/seqid=0 2024-11-09T02:23:09,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742205_1381 (size=12151) 2024-11-09T02:23:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T02:23:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:09,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:09,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119049495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:09,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:09,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:09,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119049496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:09,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:09,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:09,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119049497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:09,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:09,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119049497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:09,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:09,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:09,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:09,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119049497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:09,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:09,530 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/a583e1e39b984d0eb4173c9b644a0db2 2024-11-09T02:23:09,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/09ca3933ac7f4e8e89a189295d1b36e4 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/09ca3933ac7f4e8e89a189295d1b36e4 2024-11-09T02:23:09,537 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/09ca3933ac7f4e8e89a189295d1b36e4, entries=150, sequenceid=215, filesize=30.4 K 2024-11-09T02:23:09,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/76a07bbb007242599d2489a94d6a48e5 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/76a07bbb007242599d2489a94d6a48e5 2024-11-09T02:23:09,541 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/76a07bbb007242599d2489a94d6a48e5, entries=150, sequenceid=215, filesize=11.9 K 2024-11-09T02:23:09,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/a583e1e39b984d0eb4173c9b644a0db2 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/a583e1e39b984d0eb4173c9b644a0db2 2024-11-09T02:23:09,544 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/a583e1e39b984d0eb4173c9b644a0db2, entries=150, sequenceid=215, filesize=11.9 K 2024-11-09T02:23:09,545 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a695692878ab8811a17602b58ad8f4dd in 932ms, sequenceid=215, compaction requested=false 2024-11-09T02:23:09,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:09,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:09,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-09T02:23:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-09T02:23:09,549 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-09T02:23:09,550 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4630 sec 2024-11-09T02:23:09,551 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.4690 sec 2024-11-09T02:23:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:09,614 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:23:09,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a695692878ab8811a17602b58ad8f4dd 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T02:23:09,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109eee133d2158346bbb79d19e3c23c255b_a695692878ab8811a17602b58ad8f4dd is 50, key is test_row_0/A:col10/1731118989613/Put/seqid=0 2024-11-09T02:23:09,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742206_1382 (size=14794) 2024-11-09T02:23:09,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:09,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119049644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:09,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119049650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:09,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:09,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119049651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:09,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:09,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:09,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119049652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:09,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:09,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:09,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119049654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:09,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:09,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:09,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119049760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:09,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:09,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119049770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:09,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119049770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:09,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119049770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:09,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119049771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:09,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:09,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:09,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119049970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:09,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:09,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:09,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119049979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:09,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:09,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:09,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119049979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:09,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:09,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:09,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119049979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:09,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:09,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119049980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:09,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:09,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:09,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:10,026 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:10,029 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109eee133d2158346bbb79d19e3c23c255b_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109eee133d2158346bbb79d19e3c23c255b_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:10,030 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/7cfecf91491c4b7b89c011e4cac15430, store: [table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:10,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/7cfecf91491c4b7b89c011e4cac15430 is 175, key is test_row_0/A:col10/1731118989613/Put/seqid=0 2024-11-09T02:23:10,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742207_1383 (size=39749) 2024-11-09T02:23:10,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119050290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:10,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119050290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:10,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119050290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:10,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119050290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:10,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119050290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:10,435 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=229, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/7cfecf91491c4b7b89c011e4cac15430 2024-11-09T02:23:10,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/54e4a31f947948558b0e24d6abf2726e is 50, key is test_row_0/B:col10/1731118989613/Put/seqid=0 2024-11-09T02:23:10,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742208_1384 (size=12151) 2024-11-09T02:23:10,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:10,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119050810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:10,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119050810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:10,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119050810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:10,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119050810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:10,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119050811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:10,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:10,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:10,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:10,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/54e4a31f947948558b0e24d6abf2726e 2024-11-09T02:23:10,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/861624fbf4d84db9b52e730fb401ae03 is 50, key is test_row_0/C:col10/1731118989613/Put/seqid=0 2024-11-09T02:23:10,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742209_1385 (size=12151) 2024-11-09T02:23:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T02:23:11,240 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:23:11,241 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:23:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-09T02:23:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T02:23:11,242 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:23:11,243 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:23:11,243 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:23:11,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/861624fbf4d84db9b52e730fb401ae03 2024-11-09T02:23:11,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/7cfecf91491c4b7b89c011e4cac15430 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/7cfecf91491c4b7b89c011e4cac15430 2024-11-09T02:23:11,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/7cfecf91491c4b7b89c011e4cac15430, entries=200, sequenceid=229, filesize=38.8 K 2024-11-09T02:23:11,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/54e4a31f947948558b0e24d6abf2726e as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/54e4a31f947948558b0e24d6abf2726e 2024-11-09T02:23:11,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/54e4a31f947948558b0e24d6abf2726e, entries=150, sequenceid=229, filesize=11.9 K 2024-11-09T02:23:11,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/861624fbf4d84db9b52e730fb401ae03 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/861624fbf4d84db9b52e730fb401ae03 2024-11-09T02:23:11,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/861624fbf4d84db9b52e730fb401ae03, entries=150, sequenceid=229, filesize=11.9 K 2024-11-09T02:23:11,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for a695692878ab8811a17602b58ad8f4dd in 1664ms, sequenceid=229, compaction requested=true 2024-11-09T02:23:11,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:11,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a695692878ab8811a17602b58ad8f4dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:23:11,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:11,279 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:11,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a695692878ab8811a17602b58ad8f4dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:23:11,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:11,279 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:11,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a695692878ab8811a17602b58ad8f4dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:23:11,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:11,279 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:11,279 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:11,279 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): a695692878ab8811a17602b58ad8f4dd/A is initiating minor compaction (all files) 2024-11-09T02:23:11,280 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): a695692878ab8811a17602b58ad8f4dd/B is initiating minor compaction (all files) 2024-11-09T02:23:11,280 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a695692878ab8811a17602b58ad8f4dd/A in TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:11,280 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a695692878ab8811a17602b58ad8f4dd/B in TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:11,280 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/4ae3063f9da248afb37e294cd4da3b5d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/09ca3933ac7f4e8e89a189295d1b36e4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/7cfecf91491c4b7b89c011e4cac15430] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp, totalSize=100.0 K 2024-11-09T02:23:11,280 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/09d9e7a8dae44725a30292d6a1b8289c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/76a07bbb007242599d2489a94d6a48e5, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/54e4a31f947948558b0e24d6abf2726e] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp, totalSize=36.0 K 2024-11-09T02:23:11,280 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:11,280 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. files: [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/4ae3063f9da248afb37e294cd4da3b5d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/09ca3933ac7f4e8e89a189295d1b36e4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/7cfecf91491c4b7b89c011e4cac15430] 2024-11-09T02:23:11,280 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 09d9e7a8dae44725a30292d6a1b8289c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731118987173 2024-11-09T02:23:11,280 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ae3063f9da248afb37e294cd4da3b5d, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731118987173 2024-11-09T02:23:11,280 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 76a07bbb007242599d2489a94d6a48e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731118987284 2024-11-09T02:23:11,280 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09ca3933ac7f4e8e89a189295d1b36e4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731118987284 2024-11-09T02:23:11,280 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 54e4a31f947948558b0e24d6abf2726e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731118989495 2024-11-09T02:23:11,280 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cfecf91491c4b7b89c011e4cac15430, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731118989495 2024-11-09T02:23:11,285 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:11,286 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a695692878ab8811a17602b58ad8f4dd#B#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:11,286 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/d94275068961418d9ed09d91784d14cb is 50, key is test_row_0/B:col10/1731118989613/Put/seqid=0 2024-11-09T02:23:11,287 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110916c830cbafc14991b38c2225a33db287_a695692878ab8811a17602b58ad8f4dd store=[table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:11,290 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110916c830cbafc14991b38c2225a33db287_a695692878ab8811a17602b58ad8f4dd, store=[table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:11,290 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110916c830cbafc14991b38c2225a33db287_a695692878ab8811a17602b58ad8f4dd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:11,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742211_1387 (size=4469) 2024-11-09T02:23:11,297 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a695692878ab8811a17602b58ad8f4dd#A#compaction#322 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:11,298 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/31173651edce4211ac274c38e9d062cf is 175, key is test_row_0/A:col10/1731118989613/Put/seqid=0 2024-11-09T02:23:11,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742210_1386 (size=12697) 2024-11-09T02:23:11,307 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/d94275068961418d9ed09d91784d14cb as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/d94275068961418d9ed09d91784d14cb 2024-11-09T02:23:11,311 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a695692878ab8811a17602b58ad8f4dd/B of a695692878ab8811a17602b58ad8f4dd into d94275068961418d9ed09d91784d14cb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:11,311 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:11,311 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., storeName=a695692878ab8811a17602b58ad8f4dd/B, priority=13, startTime=1731118991279; duration=0sec 2024-11-09T02:23:11,311 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:11,311 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a695692878ab8811a17602b58ad8f4dd:B 2024-11-09T02:23:11,311 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:11,313 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:11,314 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): a695692878ab8811a17602b58ad8f4dd/C is initiating minor compaction (all files) 2024-11-09T02:23:11,314 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a695692878ab8811a17602b58ad8f4dd/C in TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:11,314 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/6bbab525d2874879b86e4d8b53cd42b3, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/a583e1e39b984d0eb4173c9b644a0db2, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/861624fbf4d84db9b52e730fb401ae03] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp, totalSize=36.0 K 2024-11-09T02:23:11,314 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bbab525d2874879b86e4d8b53cd42b3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731118987173 2024-11-09T02:23:11,314 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting a583e1e39b984d0eb4173c9b644a0db2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731118987284 2024-11-09T02:23:11,314 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 861624fbf4d84db9b52e730fb401ae03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731118989495 2024-11-09T02:23:11,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742212_1388 (size=31651) 2024-11-09T02:23:11,320 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a695692878ab8811a17602b58ad8f4dd#C#compaction#323 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:11,320 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/864103e6e1d940b4869f70589195d873 is 50, key is test_row_0/C:col10/1731118989613/Put/seqid=0 2024-11-09T02:23:11,324 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/31173651edce4211ac274c38e9d062cf as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/31173651edce4211ac274c38e9d062cf 2024-11-09T02:23:11,330 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a695692878ab8811a17602b58ad8f4dd/A of a695692878ab8811a17602b58ad8f4dd into 31173651edce4211ac274c38e9d062cf(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:11,330 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:11,330 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., storeName=a695692878ab8811a17602b58ad8f4dd/A, priority=13, startTime=1731118991279; duration=0sec 2024-11-09T02:23:11,330 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:11,330 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a695692878ab8811a17602b58ad8f4dd:A 2024-11-09T02:23:11,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742213_1389 (size=12697) 2024-11-09T02:23:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T02:23:11,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-09T02:23:11,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:11,395 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing a695692878ab8811a17602b58ad8f4dd 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-09T02:23:11,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110967df1f89d4004dd6a173ca9da247f46a_a695692878ab8811a17602b58ad8f4dd is 50, key is test_row_0/A:col10/1731118989650/Put/seqid=0 2024-11-09T02:23:11,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742214_1390 (size=12304) 2024-11-09T02:23:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T02:23:11,737 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/864103e6e1d940b4869f70589195d873 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/864103e6e1d940b4869f70589195d873 2024-11-09T02:23:11,741 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a695692878ab8811a17602b58ad8f4dd/C of a695692878ab8811a17602b58ad8f4dd into 864103e6e1d940b4869f70589195d873(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:11,741 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:11,741 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., storeName=a695692878ab8811a17602b58ad8f4dd/C, priority=13, startTime=1731118991279; duration=0sec 2024-11-09T02:23:11,741 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:11,741 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a695692878ab8811a17602b58ad8f4dd:C 2024-11-09T02:23:11,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:11,808 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110967df1f89d4004dd6a173ca9da247f46a_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110967df1f89d4004dd6a173ca9da247f46a_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:11,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/60cbe1f4d28c45bcae5a17cdfa6fcfde, store: [table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:11,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/60cbe1f4d28c45bcae5a17cdfa6fcfde is 175, key is test_row_0/A:col10/1731118989650/Put/seqid=0 2024-11-09T02:23:11,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742215_1391 (size=31105) 2024-11-09T02:23:11,822 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/60cbe1f4d28c45bcae5a17cdfa6fcfde 2024-11-09T02:23:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:11,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:11,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/d5b602c0ebc34eb0845759c6a31c957f is 50, key is test_row_0/B:col10/1731118989650/Put/seqid=0 2024-11-09T02:23:11,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742216_1392 (size=12151) 2024-11-09T02:23:11,840 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/d5b602c0ebc34eb0845759c6a31c957f 2024-11-09T02:23:11,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/798b3661a04448a4b53f5bd207b53713 is 50, key is test_row_0/C:col10/1731118989650/Put/seqid=0 2024-11-09T02:23:11,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:11,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119051845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:11,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:11,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:11,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:11,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742217_1393 (size=12151) 2024-11-09T02:23:11,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119051847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:11,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119051847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:11,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:11,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:11,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:11,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:11,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119051847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:11,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:11,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:11,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:11,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:11,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:11,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119051848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:11,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T02:23:11,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:11,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119051949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:11,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:11,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:11,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:11,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119051960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:11,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:11,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119051960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:11,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:11,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:11,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:11,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:11,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:11,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119051960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:11,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:11,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:11,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:11,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:11,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119051960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:11,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:11,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:11,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:12,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119052159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:12,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:12,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119052170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119052170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119052170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:12,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119052170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:12,250 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/798b3661a04448a4b53f5bd207b53713 2024-11-09T02:23:12,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/60cbe1f4d28c45bcae5a17cdfa6fcfde as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/60cbe1f4d28c45bcae5a17cdfa6fcfde 2024-11-09T02:23:12,256 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/60cbe1f4d28c45bcae5a17cdfa6fcfde, entries=150, sequenceid=255, filesize=30.4 K 2024-11-09T02:23:12,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/d5b602c0ebc34eb0845759c6a31c957f as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/d5b602c0ebc34eb0845759c6a31c957f 2024-11-09T02:23:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,260 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/d5b602c0ebc34eb0845759c6a31c957f, entries=150, sequenceid=255, filesize=11.9 K 2024-11-09T02:23:12,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/798b3661a04448a4b53f5bd207b53713 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/798b3661a04448a4b53f5bd207b53713 2024-11-09T02:23:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,264 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/798b3661a04448a4b53f5bd207b53713, entries=150, sequenceid=255, filesize=11.9 K 2024-11-09T02:23:12,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,265 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for a695692878ab8811a17602b58ad8f4dd in 869ms, sequenceid=255, compaction requested=false 2024-11-09T02:23:12,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:12,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:12,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-09T02:23:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-09T02:23:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,267 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-09T02:23:12,267 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0230 sec 2024-11-09T02:23:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,269 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.0270 sec 2024-11-09T02:23:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T02:23:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,380 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:23:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,381 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:23:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-09T02:23:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T02:23:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,383 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:23:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,384 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:23:12,384 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:23:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:12,482 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:23:12,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a695692878ab8811a17602b58ad8f4dd 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-09T02:23:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T02:23:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099597b021518f4716924a63075d827bcc_a695692878ab8811a17602b58ad8f4dd is 50, key is test_row_0/A:col10/1731118991846/Put/seqid=0 2024-11-09T02:23:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742218_1394 (size=12454) 2024-11-09T02:23:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T02:23:12,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:12,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:12,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:12,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:12,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:12,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119052541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:12,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119052545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:12,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119052547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:12,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:12,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119052549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119052548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:12,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119052660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:12,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:12,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119052670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:12,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:12,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119052670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119052670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119052670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:12,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T02:23:12,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:12,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:12,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:12,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:12,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:12,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T02:23:12,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T02:23:12,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:12,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:12,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:12,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:12,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:12,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:12,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119052870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:12,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:12,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119052889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:12,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:12,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119052890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:12,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:12,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119052890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:12,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:12,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119052890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:12,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:12,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:12,912 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:12,915 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099597b021518f4716924a63075d827bcc_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099597b021518f4716924a63075d827bcc_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:12,916 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/f815d44b6d2742d892c0ed568f95a819, store: [table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:12,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/f815d44b6d2742d892c0ed568f95a819 is 175, key is test_row_0/A:col10/1731118991846/Put/seqid=0 2024-11-09T02:23:12,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742219_1395 (size=31255) 2024-11-09T02:23:12,920 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=270, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/f815d44b6d2742d892c0ed568f95a819 2024-11-09T02:23:12,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/8feb2cf6f9f44aeeaf1c88506400068c is 50, key is test_row_0/B:col10/1731118991846/Put/seqid=0 2024-11-09T02:23:12,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742220_1396 (size=12301) 2024-11-09T02:23:12,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T02:23:12,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:12,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:12,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:12,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:12,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:12,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:13,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T02:23:13,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T02:23:13,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:13,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:13,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:13,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:13,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:13,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:13,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:13,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119053180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:13,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:13,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:13,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:13,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:13,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119053199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:13,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:13,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:13,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:13,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119053200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:13,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:13,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:13,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:13,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:13,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119053200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:13,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:13,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:13,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:13,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:13,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:13,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119053210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:13,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:13,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:13,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:13,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T02:23:13,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:13,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:13,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:13,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:13,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:13,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:13,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/8feb2cf6f9f44aeeaf1c88506400068c 2024-11-09T02:23:13,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/8145c8c5f9df4b39a74e826e9193dc52 is 50, key is test_row_0/C:col10/1731118991846/Put/seqid=0 2024-11-09T02:23:13,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742221_1397 (size=12301) 2024-11-09T02:23:13,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T02:23:13,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:13,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:13,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:13,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:13,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:13,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:13,498 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T02:23:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T02:23:13,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T02:23:13,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:13,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. as already flushing 2024-11-09T02:23:13,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:13,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:13,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:13,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:13,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:13,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119053690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:13,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:13,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:13,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:13,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119053710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:13,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:13,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:13,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119053710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:13,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:13,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:13,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:13,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:13,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119053710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:13,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:13,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119053719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:13,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:13,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:13,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:13,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/8145c8c5f9df4b39a74e826e9193dc52 2024-11-09T02:23:13,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/f815d44b6d2742d892c0ed568f95a819 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/f815d44b6d2742d892c0ed568f95a819 2024-11-09T02:23:13,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/f815d44b6d2742d892c0ed568f95a819, entries=150, sequenceid=270, filesize=30.5 K 2024-11-09T02:23:13,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/8feb2cf6f9f44aeeaf1c88506400068c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/8feb2cf6f9f44aeeaf1c88506400068c 2024-11-09T02:23:13,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/8feb2cf6f9f44aeeaf1c88506400068c, entries=150, sequenceid=270, filesize=12.0 K 2024-11-09T02:23:13,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/8145c8c5f9df4b39a74e826e9193dc52 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/8145c8c5f9df4b39a74e826e9193dc52 2024-11-09T02:23:13,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/8145c8c5f9df4b39a74e826e9193dc52, entries=150, sequenceid=270, filesize=12.0 K 2024-11-09T02:23:13,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for a695692878ab8811a17602b58ad8f4dd in 1284ms, sequenceid=270, compaction requested=true 2024-11-09T02:23:13,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:13,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a695692878ab8811a17602b58ad8f4dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:23:13,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:13,766 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:13,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a695692878ab8811a17602b58ad8f4dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:23:13,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:13,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a695692878ab8811a17602b58ad8f4dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:23:13,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T02:23:13,767 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:13,767 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:13,767 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): a695692878ab8811a17602b58ad8f4dd/B is initiating minor compaction (all files) 2024-11-09T02:23:13,768 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a695692878ab8811a17602b58ad8f4dd/B in TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:13,768 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/d94275068961418d9ed09d91784d14cb, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/d5b602c0ebc34eb0845759c6a31c957f, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/8feb2cf6f9f44aeeaf1c88506400068c] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp, totalSize=36.3 K 2024-11-09T02:23:13,768 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94011 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:13,768 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): a695692878ab8811a17602b58ad8f4dd/A is initiating minor compaction (all files) 2024-11-09T02:23:13,768 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a695692878ab8811a17602b58ad8f4dd/A in TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:13,768 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/31173651edce4211ac274c38e9d062cf, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/60cbe1f4d28c45bcae5a17cdfa6fcfde, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/f815d44b6d2742d892c0ed568f95a819] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp, totalSize=91.8 K 2024-11-09T02:23:13,768 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:13,768 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. files: [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/31173651edce4211ac274c38e9d062cf, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/60cbe1f4d28c45bcae5a17cdfa6fcfde, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/f815d44b6d2742d892c0ed568f95a819] 2024-11-09T02:23:13,768 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting d94275068961418d9ed09d91784d14cb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731118989495 2024-11-09T02:23:13,768 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31173651edce4211ac274c38e9d062cf, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731118989495 2024-11-09T02:23:13,770 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting d5b602c0ebc34eb0845759c6a31c957f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731118989641 2024-11-09T02:23:13,770 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60cbe1f4d28c45bcae5a17cdfa6fcfde, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731118989641 2024-11-09T02:23:13,770 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 8feb2cf6f9f44aeeaf1c88506400068c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731118991845 2024-11-09T02:23:13,772 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting f815d44b6d2742d892c0ed568f95a819, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731118991845 2024-11-09T02:23:13,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T02:23:13,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:13,775 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing a695692878ab8811a17602b58ad8f4dd 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-09T02:23:13,785 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:13,788 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a695692878ab8811a17602b58ad8f4dd#B#compaction#331 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:13,788 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/cc59d247146e403897dfa49adc1874c4 is 50, key is test_row_0/B:col10/1731118991846/Put/seqid=0 2024-11-09T02:23:13,793 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109e9b50cdb390b4c5bb2533217a76af433_a695692878ab8811a17602b58ad8f4dd store=[table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:13,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110900d3af5843634295921ef5ddf2e8cd01_a695692878ab8811a17602b58ad8f4dd is 50, key is test_row_0/A:col10/1731118992542/Put/seqid=0 2024-11-09T02:23:13,797 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109e9b50cdb390b4c5bb2533217a76af433_a695692878ab8811a17602b58ad8f4dd, store=[table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:13,798 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e9b50cdb390b4c5bb2533217a76af433_a695692878ab8811a17602b58ad8f4dd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:13,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742222_1398 (size=12949) 2024-11-09T02:23:13,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742223_1399 (size=12454) 2024-11-09T02:23:13,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742224_1400 (size=4469) 2024-11-09T02:23:14,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,214 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/cc59d247146e403897dfa49adc1874c4 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/cc59d247146e403897dfa49adc1874c4 2024-11-09T02:23:14,215 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110900d3af5843634295921ef5ddf2e8cd01_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110900d3af5843634295921ef5ddf2e8cd01_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:14,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/ff2764dcac6c4bb981e734f416909a7f, store: [table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:14,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/ff2764dcac6c4bb981e734f416909a7f is 175, key is test_row_0/A:col10/1731118992542/Put/seqid=0 2024-11-09T02:23:14,218 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a695692878ab8811a17602b58ad8f4dd#A#compaction#330 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:14,218 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/ecd39462ba894d31b952ac3eccceca10 is 175, key is test_row_0/A:col10/1731118991846/Put/seqid=0 2024-11-09T02:23:14,232 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a695692878ab8811a17602b58ad8f4dd/B of a695692878ab8811a17602b58ad8f4dd into cc59d247146e403897dfa49adc1874c4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:14,232 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:14,232 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., storeName=a695692878ab8811a17602b58ad8f4dd/B, priority=13, startTime=1731118993766; duration=0sec 2024-11-09T02:23:14,232 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:14,232 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a695692878ab8811a17602b58ad8f4dd:B 2024-11-09T02:23:14,232 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:14,233 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:14,233 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): a695692878ab8811a17602b58ad8f4dd/C is initiating minor compaction (all files) 2024-11-09T02:23:14,233 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a695692878ab8811a17602b58ad8f4dd/C in TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:14,233 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/864103e6e1d940b4869f70589195d873, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/798b3661a04448a4b53f5bd207b53713, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/8145c8c5f9df4b39a74e826e9193dc52] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp, totalSize=36.3 K 2024-11-09T02:23:14,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742225_1401 (size=31255) 2024-11-09T02:23:14,234 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 864103e6e1d940b4869f70589195d873, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731118989495 2024-11-09T02:23:14,234 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/ff2764dcac6c4bb981e734f416909a7f 2024-11-09T02:23:14,234 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 798b3661a04448a4b53f5bd207b53713, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731118989641 2024-11-09T02:23:14,235 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 8145c8c5f9df4b39a74e826e9193dc52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731118991845 2024-11-09T02:23:14,246 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a695692878ab8811a17602b58ad8f4dd#C#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:14,246 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/9a834208220c498482e3f6298bff5227 is 50, key is test_row_0/C:col10/1731118991846/Put/seqid=0 2024-11-09T02:23:14,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/93aac7fff4e54b03be3125aa8e39836d is 50, key is test_row_0/B:col10/1731118992542/Put/seqid=0 2024-11-09T02:23:14,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742226_1402 (size=31903) 2024-11-09T02:23:14,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742227_1403 (size=12949) 2024-11-09T02:23:14,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742228_1404 (size=12301) 2024-11-09T02:23:14,261 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/93aac7fff4e54b03be3125aa8e39836d 2024-11-09T02:23:14,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/5f5321fd63e043d28e5ba11214e5ca85 is 50, key is test_row_0/C:col10/1731118992542/Put/seqid=0 2024-11-09T02:23:14,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742229_1405 (size=12301) 2024-11-09T02:23:14,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T02:23:14,654 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/ecd39462ba894d31b952ac3eccceca10 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/ecd39462ba894d31b952ac3eccceca10 2024-11-09T02:23:14,659 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a695692878ab8811a17602b58ad8f4dd/A of a695692878ab8811a17602b58ad8f4dd into ecd39462ba894d31b952ac3eccceca10(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:14,659 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:14,659 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., storeName=a695692878ab8811a17602b58ad8f4dd/A, priority=13, startTime=1731118993766; duration=0sec 2024-11-09T02:23:14,659 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:14,659 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a695692878ab8811a17602b58ad8f4dd:A 2024-11-09T02:23:14,662 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/9a834208220c498482e3f6298bff5227 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/9a834208220c498482e3f6298bff5227 2024-11-09T02:23:14,668 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a695692878ab8811a17602b58ad8f4dd/C of a695692878ab8811a17602b58ad8f4dd into 9a834208220c498482e3f6298bff5227(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:14,668 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:14,668 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., storeName=a695692878ab8811a17602b58ad8f4dd/C, priority=13, startTime=1731118993766; duration=0sec 2024-11-09T02:23:14,668 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:14,668 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a695692878ab8811a17602b58ad8f4dd:C 2024-11-09T02:23:14,683 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/5f5321fd63e043d28e5ba11214e5ca85 2024-11-09T02:23:14,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/ff2764dcac6c4bb981e734f416909a7f as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/ff2764dcac6c4bb981e734f416909a7f 2024-11-09T02:23:14,691 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/ff2764dcac6c4bb981e734f416909a7f, entries=150, sequenceid=292, filesize=30.5 K 2024-11-09T02:23:14,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/93aac7fff4e54b03be3125aa8e39836d as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/93aac7fff4e54b03be3125aa8e39836d 2024-11-09T02:23:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,695 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/93aac7fff4e54b03be3125aa8e39836d, entries=150, sequenceid=292, filesize=12.0 K 2024-11-09T02:23:14,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/5f5321fd63e043d28e5ba11214e5ca85 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/5f5321fd63e043d28e5ba11214e5ca85 2024-11-09T02:23:14,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,699 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/5f5321fd63e043d28e5ba11214e5ca85, entries=150, sequenceid=292, filesize=12.0 K 2024-11-09T02:23:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,700 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for a695692878ab8811a17602b58ad8f4dd in 925ms, sequenceid=292, compaction requested=false 2024-11-09T02:23:14,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:14,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:14,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-09T02:23:14,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-09T02:23:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,703 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-09T02:23:14,703 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3180 sec 2024-11-09T02:23:14,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,706 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.3230 sec 2024-11-09T02:23:14,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:14,828 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:23:14,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a695692878ab8811a17602b58ad8f4dd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T02:23:14,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109aa9dd900384740c988d00564a67f341d_a695692878ab8811a17602b58ad8f4dd is 50, key is test_row_0/A:col10/1731118994773/Put/seqid=0 2024-11-09T02:23:14,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742231_1407 (size=25158) 2024-11-09T02:23:14,866 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:14,872 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109aa9dd900384740c988d00564a67f341d_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109aa9dd900384740c988d00564a67f341d_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:14,873 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/fe1fcb2135e14a598b30d67f1c23326a, store: [table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:14,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/fe1fcb2135e14a598b30d67f1c23326a is 175, key is test_row_0/A:col10/1731118994773/Put/seqid=0 2024-11-09T02:23:14,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742230_1406 (size=74795) 2024-11-09T02:23:14,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:14,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119054906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:14,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:14,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119054907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:14,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:14,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:14,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119054908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:14,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:14,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:14,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:14,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119054910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:14,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:14,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:14,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:14,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:14,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119054911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:14,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:14,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:14,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:14,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:14,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:14,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:14,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:14,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:14,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:15,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:15,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119055029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:15,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119055030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:15,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119055030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:15,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119055030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:15,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119055030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:15,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119055240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:15,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119055240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119055240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:15,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:15,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:15,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119055240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:15,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:15,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:15,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119055240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:15,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:15,277 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/fe1fcb2135e14a598b30d67f1c23326a 2024-11-09T02:23:15,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/6490b7736d5544728e0acc7024f56d7c is 50, key is test_row_0/B:col10/1731118994773/Put/seqid=0 2024-11-09T02:23:15,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742232_1408 (size=12301) 2024-11-09T02:23:15,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:15,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119055549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:15,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:15,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119055550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:15,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:15,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119055550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:15,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:15,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:15,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119055550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:15,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:15,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:15,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119055550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:15,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:15,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:15,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/6490b7736d5544728e0acc7024f56d7c 2024-11-09T02:23:15,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/3f488be2cae3488386c99f044f5afd5f is 50, key is test_row_0/C:col10/1731118994773/Put/seqid=0 2024-11-09T02:23:15,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742233_1409 (size=12301) 2024-11-09T02:23:16,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:16,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48460 deadline: 1731119056059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:16,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:16,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48486 deadline: 1731119056060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:16,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:16,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48478 deadline: 1731119056070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:16,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:16,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:16,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:16,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:16,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48436 deadline: 1731119056070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:16,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:16,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:16,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:16,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:16,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48458 deadline: 1731119056070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:16,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/3f488be2cae3488386c99f044f5afd5f 2024-11-09T02:23:16,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/fe1fcb2135e14a598b30d67f1c23326a as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/fe1fcb2135e14a598b30d67f1c23326a 2024-11-09T02:23:16,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/fe1fcb2135e14a598b30d67f1c23326a, entries=400, sequenceid=306, filesize=73.0 K 2024-11-09T02:23:16,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/6490b7736d5544728e0acc7024f56d7c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/6490b7736d5544728e0acc7024f56d7c 2024-11-09T02:23:16,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/6490b7736d5544728e0acc7024f56d7c, entries=150, sequenceid=306, filesize=12.0 K 2024-11-09T02:23:16,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/3f488be2cae3488386c99f044f5afd5f as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/3f488be2cae3488386c99f044f5afd5f 2024-11-09T02:23:16,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/3f488be2cae3488386c99f044f5afd5f, entries=150, sequenceid=306, filesize=12.0 K 2024-11-09T02:23:16,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a695692878ab8811a17602b58ad8f4dd in 1282ms, sequenceid=306, compaction requested=true 2024-11-09T02:23:16,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:16,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a695692878ab8811a17602b58ad8f4dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:23:16,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:16,111 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:16,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a695692878ab8811a17602b58ad8f4dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:23:16,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:16,111 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:16,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a695692878ab8811a17602b58ad8f4dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:23:16,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:16,111 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 137953 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:16,111 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:16,111 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): a695692878ab8811a17602b58ad8f4dd/B is initiating minor compaction (all files) 2024-11-09T02:23:16,111 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): a695692878ab8811a17602b58ad8f4dd/A is initiating minor compaction (all files) 2024-11-09T02:23:16,111 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a695692878ab8811a17602b58ad8f4dd/A in TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:16,111 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a695692878ab8811a17602b58ad8f4dd/B in TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:16,111 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/ecd39462ba894d31b952ac3eccceca10, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/ff2764dcac6c4bb981e734f416909a7f, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/fe1fcb2135e14a598b30d67f1c23326a] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp, totalSize=134.7 K 2024-11-09T02:23:16,111 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/cc59d247146e403897dfa49adc1874c4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/93aac7fff4e54b03be3125aa8e39836d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/6490b7736d5544728e0acc7024f56d7c] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp, totalSize=36.7 K 2024-11-09T02:23:16,111 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:16,112 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. files: [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/ecd39462ba894d31b952ac3eccceca10, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/ff2764dcac6c4bb981e734f416909a7f, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/fe1fcb2135e14a598b30d67f1c23326a] 2024-11-09T02:23:16,112 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting cc59d247146e403897dfa49adc1874c4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731118991845 2024-11-09T02:23:16,112 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecd39462ba894d31b952ac3eccceca10, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731118991845 2024-11-09T02:23:16,112 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 93aac7fff4e54b03be3125aa8e39836d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731118992542 2024-11-09T02:23:16,112 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff2764dcac6c4bb981e734f416909a7f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731118992542 2024-11-09T02:23:16,112 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 6490b7736d5544728e0acc7024f56d7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731118994773 2024-11-09T02:23:16,112 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe1fcb2135e14a598b30d67f1c23326a, keycount=400, bloomtype=ROW, size=73.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731118994768 2024-11-09T02:23:16,117 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:16,118 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a695692878ab8811a17602b58ad8f4dd#B#compaction#339 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:16,118 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109ff3c7e59addb4c479a1f07bbf703794c_a695692878ab8811a17602b58ad8f4dd store=[table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:16,118 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/9824d6d2f322489ea39d127c29436611 is 50, key is test_row_0/B:col10/1731118994773/Put/seqid=0 2024-11-09T02:23:16,120 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109ff3c7e59addb4c479a1f07bbf703794c_a695692878ab8811a17602b58ad8f4dd, store=[table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:16,120 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ff3c7e59addb4c479a1f07bbf703794c_a695692878ab8811a17602b58ad8f4dd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:16,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742234_1410 (size=13051) 2024-11-09T02:23:16,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742235_1411 (size=4469) 2024-11-09T02:23:16,132 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a695692878ab8811a17602b58ad8f4dd#A#compaction#340 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:16,133 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/02c37775bc424887a07de6c90ee3e1f9 is 175, key is test_row_0/A:col10/1731118994773/Put/seqid=0 2024-11-09T02:23:16,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742236_1412 (size=32005) 2024-11-09T02:23:16,146 INFO [Thread-1493 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1493. 2024-11-09T02:23:16,146 DEBUG [Thread-1493 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:23:16,146 DEBUG [Thread-1493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:16,146 DEBUG [Thread-1493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:16,146 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:23:16,147 INFO [Thread-1489 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1489. 2024-11-09T02:23:16,147 DEBUG [Thread-1489 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:23:16,147 DEBUG [Thread-1489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:16,147 DEBUG [Thread-1489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:16,147 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:23:16,151 INFO [Thread-1495 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1495. 2024-11-09T02:23:16,151 DEBUG [Thread-1495 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:23:16,151 DEBUG [Thread-1495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:16,151 DEBUG [Thread-1495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:16,151 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:23:16,153 INFO [Thread-1491 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1491. 2024-11-09T02:23:16,153 DEBUG [Thread-1491 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:23:16,153 DEBUG [Thread-1491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:16,153 DEBUG [Thread-1491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:16,153 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:23:16,154 INFO [Thread-1487 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1487. 2024-11-09T02:23:16,154 DEBUG [Thread-1487 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:23:16,154 DEBUG [Thread-1487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:16,154 DEBUG [Thread-1487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:16,154 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:23:16,537 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/9824d6d2f322489ea39d127c29436611 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/9824d6d2f322489ea39d127c29436611 2024-11-09T02:23:16,545 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a695692878ab8811a17602b58ad8f4dd/B of a695692878ab8811a17602b58ad8f4dd into 9824d6d2f322489ea39d127c29436611(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:16,545 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:16,545 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., storeName=a695692878ab8811a17602b58ad8f4dd/B, priority=13, startTime=1731118996111; duration=0sec 2024-11-09T02:23:16,545 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:16,545 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a695692878ab8811a17602b58ad8f4dd:B 2024-11-09T02:23:16,545 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:16,547 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:16,547 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): a695692878ab8811a17602b58ad8f4dd/C is initiating minor compaction (all files) 2024-11-09T02:23:16,547 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a695692878ab8811a17602b58ad8f4dd/C in TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:16,547 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/9a834208220c498482e3f6298bff5227, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/5f5321fd63e043d28e5ba11214e5ca85, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/3f488be2cae3488386c99f044f5afd5f] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp, totalSize=36.7 K 2024-11-09T02:23:16,547 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/02c37775bc424887a07de6c90ee3e1f9 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/02c37775bc424887a07de6c90ee3e1f9 2024-11-09T02:23:16,547 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a834208220c498482e3f6298bff5227, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731118991845 2024-11-09T02:23:16,548 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f5321fd63e043d28e5ba11214e5ca85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731118992542 2024-11-09T02:23:16,548 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f488be2cae3488386c99f044f5afd5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731118994773 2024-11-09T02:23:16,552 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a695692878ab8811a17602b58ad8f4dd/A of a695692878ab8811a17602b58ad8f4dd into 02c37775bc424887a07de6c90ee3e1f9(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:16,552 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:16,553 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., storeName=a695692878ab8811a17602b58ad8f4dd/A, priority=13, startTime=1731118996110; duration=0sec 2024-11-09T02:23:16,553 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:16,553 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a695692878ab8811a17602b58ad8f4dd:A 2024-11-09T02:23:16,555 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a695692878ab8811a17602b58ad8f4dd#C#compaction#341 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:16,555 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/28a84705ffd1465aa40b3aebbca21cb7 is 50, key is test_row_0/C:col10/1731118994773/Put/seqid=0 2024-11-09T02:23:16,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742237_1413 (size=13051) 2024-11-09T02:23:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T02:23:16,560 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:23:16,962 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/28a84705ffd1465aa40b3aebbca21cb7 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/28a84705ffd1465aa40b3aebbca21cb7 2024-11-09T02:23:16,966 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a695692878ab8811a17602b58ad8f4dd/C of a695692878ab8811a17602b58ad8f4dd into 28a84705ffd1465aa40b3aebbca21cb7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:16,966 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:16,966 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., storeName=a695692878ab8811a17602b58ad8f4dd/C, priority=13, startTime=1731118996111; duration=0sec 2024-11-09T02:23:16,966 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:16,967 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a695692878ab8811a17602b58ad8f4dd:C 2024-11-09T02:23:17,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:17,085 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:23:17,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a695692878ab8811a17602b58ad8f4dd 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-09T02:23:17,086 INFO [Thread-1482 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1482. 2024-11-09T02:23:17,086 INFO [Thread-1480 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1480. 2024-11-09T02:23:17,086 DEBUG [Thread-1482 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:23:17,086 DEBUG [Thread-1480 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:23:17,086 DEBUG [Thread-1482 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:17,086 DEBUG [Thread-1480 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:17,086 DEBUG [Thread-1480 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:17,086 DEBUG [Thread-1482 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:17,086 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:23:17,086 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:23:17,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109238875633abd446888936984dbdefff7_a695692878ab8811a17602b58ad8f4dd is 50, key is test_row_0/A:col10/1731118997080/Put/seqid=0 2024-11-09T02:23:17,091 INFO [Thread-1476 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1476. 2024-11-09T02:23:17,091 INFO [Thread-1484 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1484. 2024-11-09T02:23:17,091 DEBUG [Thread-1476 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:23:17,091 DEBUG [Thread-1484 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:23:17,092 DEBUG [Thread-1476 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:17,092 DEBUG [Thread-1484 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:17,092 DEBUG [Thread-1484 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:17,092 DEBUG [Thread-1476 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:17,092 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:23:17,092 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:23:17,092 INFO [Thread-1478 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1478. 2024-11-09T02:23:17,092 DEBUG [Thread-1478 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:23:17,092 DEBUG [Thread-1478 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:17,092 DEBUG [Thread-1478 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:17,092 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:23:17,092 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T02:23:17,092 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-09T02:23:17,092 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-09T02:23:17,092 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-09T02:23:17,092 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-09T02:23:17,092 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-09T02:23:17,092 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T02:23:17,092 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T02:23:17,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1342 2024-11-09T02:23:17,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4026 rows 2024-11-09T02:23:17,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1346 2024-11-09T02:23:17,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4035 rows 2024-11-09T02:23:17,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1342 2024-11-09T02:23:17,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4026 rows 2024-11-09T02:23:17,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1344 2024-11-09T02:23:17,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4031 rows 2024-11-09T02:23:17,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1340 2024-11-09T02:23:17,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4016 rows 2024-11-09T02:23:17,093 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T02:23:17,093 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:23:17,093 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:17,093 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:17,093 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:23:17,094 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T02:23:17,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T02:23:17,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T02:23:17,096 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118997096"}]},"ts":"1731118997096"} 2024-11-09T02:23:17,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742238_1414 (size=12454) 2024-11-09T02:23:17,098 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T02:23:17,098 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T02:23:17,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T02:23:17,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a695692878ab8811a17602b58ad8f4dd, UNASSIGN}] 2024-11-09T02:23:17,100 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a695692878ab8811a17602b58ad8f4dd, UNASSIGN 2024-11-09T02:23:17,100 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=a695692878ab8811a17602b58ad8f4dd, regionState=CLOSING, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:17,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a695692878ab8811a17602b58ad8f4dd, UNASSIGN because future has completed 2024-11-09T02:23:17,102 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T02:23:17,102 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223}] 2024-11-09T02:23:17,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T02:23:17,254 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:17,254 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T02:23:17,254 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing a695692878ab8811a17602b58ad8f4dd, disabling compactions & flushes 2024-11-09T02:23:17,255 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:17,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T02:23:17,499 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:17,507 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109238875633abd446888936984dbdefff7_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109238875633abd446888936984dbdefff7_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:17,509 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/171fcde653ee4153be78c47ffabf7540, store: [table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:17,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/171fcde653ee4153be78c47ffabf7540 is 175, key is test_row_0/A:col10/1731118997080/Put/seqid=0 2024-11-09T02:23:17,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742239_1415 (size=31255) 2024-11-09T02:23:17,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T02:23:17,914 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/171fcde653ee4153be78c47ffabf7540 2024-11-09T02:23:17,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/689ff382417b47ea83bf1009662baf8f is 50, key is test_row_0/B:col10/1731118997080/Put/seqid=0 2024-11-09T02:23:17,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742240_1416 (size=12301) 2024-11-09T02:23:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T02:23:18,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/689ff382417b47ea83bf1009662baf8f 2024-11-09T02:23:18,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/daa85faa0d01414c95e2e055538c5bab is 50, key is test_row_0/C:col10/1731118997080/Put/seqid=0 2024-11-09T02:23:18,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742241_1417 (size=12301) 2024-11-09T02:23:18,464 WARN [regionserver/53f286d3af3b:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-11-09T02:23:18,474 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/74b4ce6bbde54972af252da329afa084, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/8a6d0d322bdb495f9c6732027dbf6e03, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/6637d17800b94ec188bbad4822c5dbc9, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/3502cb78c75f45b388f75d621e193036, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/6a65bd5aff4b402880436f5744d15ea6, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/3b0c724f3746434bb229e0c3e3e099d3, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/13995700c51c4b7e8592c3770357de05, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/72fd3892aa5d4e5aa274353f6811a36f, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/129b7d4c2ee240889da3e9861e4a9412, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/ba62ea1f1a9d450988703e8e08ce7ce6, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/744b66d8822e4323a72aba64888917ca, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/09e2ea63543248e099035bae98ed135b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/56e9de76ad744c90b0037f77d0bff053, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/4ae3063f9da248afb37e294cd4da3b5d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/09ca3933ac7f4e8e89a189295d1b36e4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/7cfecf91491c4b7b89c011e4cac15430, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/31173651edce4211ac274c38e9d062cf, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/60cbe1f4d28c45bcae5a17cdfa6fcfde, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/ecd39462ba894d31b952ac3eccceca10, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/f815d44b6d2742d892c0ed568f95a819, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/ff2764dcac6c4bb981e734f416909a7f, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/fe1fcb2135e14a598b30d67f1c23326a] to archive 2024-11-09T02:23:18,475 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T02:23:18,477 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/74b4ce6bbde54972af252da329afa084 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/74b4ce6bbde54972af252da329afa084 2024-11-09T02:23:18,478 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/8a6d0d322bdb495f9c6732027dbf6e03 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/8a6d0d322bdb495f9c6732027dbf6e03 2024-11-09T02:23:18,479 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/6637d17800b94ec188bbad4822c5dbc9 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/6637d17800b94ec188bbad4822c5dbc9 2024-11-09T02:23:18,480 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/3502cb78c75f45b388f75d621e193036 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/3502cb78c75f45b388f75d621e193036 2024-11-09T02:23:18,481 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/6a65bd5aff4b402880436f5744d15ea6 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/6a65bd5aff4b402880436f5744d15ea6 2024-11-09T02:23:18,482 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/3b0c724f3746434bb229e0c3e3e099d3 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/3b0c724f3746434bb229e0c3e3e099d3 2024-11-09T02:23:18,482 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/13995700c51c4b7e8592c3770357de05 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/13995700c51c4b7e8592c3770357de05 2024-11-09T02:23:18,483 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/72fd3892aa5d4e5aa274353f6811a36f to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/72fd3892aa5d4e5aa274353f6811a36f 2024-11-09T02:23:18,484 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/129b7d4c2ee240889da3e9861e4a9412 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/129b7d4c2ee240889da3e9861e4a9412 2024-11-09T02:23:18,485 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/ba62ea1f1a9d450988703e8e08ce7ce6 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/ba62ea1f1a9d450988703e8e08ce7ce6 2024-11-09T02:23:18,485 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/744b66d8822e4323a72aba64888917ca to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/744b66d8822e4323a72aba64888917ca 2024-11-09T02:23:18,486 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/09e2ea63543248e099035bae98ed135b to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/09e2ea63543248e099035bae98ed135b 2024-11-09T02:23:18,487 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/56e9de76ad744c90b0037f77d0bff053 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/56e9de76ad744c90b0037f77d0bff053 2024-11-09T02:23:18,487 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/4ae3063f9da248afb37e294cd4da3b5d to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/4ae3063f9da248afb37e294cd4da3b5d 2024-11-09T02:23:18,488 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/09ca3933ac7f4e8e89a189295d1b36e4 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/09ca3933ac7f4e8e89a189295d1b36e4 2024-11-09T02:23:18,489 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/7cfecf91491c4b7b89c011e4cac15430 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/7cfecf91491c4b7b89c011e4cac15430 2024-11-09T02:23:18,491 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/31173651edce4211ac274c38e9d062cf to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/31173651edce4211ac274c38e9d062cf 2024-11-09T02:23:18,492 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/60cbe1f4d28c45bcae5a17cdfa6fcfde to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/60cbe1f4d28c45bcae5a17cdfa6fcfde 2024-11-09T02:23:18,493 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/ecd39462ba894d31b952ac3eccceca10 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/ecd39462ba894d31b952ac3eccceca10 2024-11-09T02:23:18,494 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/f815d44b6d2742d892c0ed568f95a819 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/f815d44b6d2742d892c0ed568f95a819 2024-11-09T02:23:18,495 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/ff2764dcac6c4bb981e734f416909a7f to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/ff2764dcac6c4bb981e734f416909a7f 2024-11-09T02:23:18,496 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/fe1fcb2135e14a598b30d67f1c23326a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/fe1fcb2135e14a598b30d67f1c23326a 2024-11-09T02:23:18,499 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/b24c730f62a643b5a9c23acd9ba14919, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/7b6a921ee6a840d186a8a30a8e280041, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/27e420f2e3e6416ebd57102df880a826, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/12ba5e23c2cc4669856c78a4ad4f48a3, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/6118d4a29a044ae389a43992c14cc198, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/5c006ffcdbb547a5b9458b70114dc908, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/0f8b2c0fe4424c96946161deddec14b5, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/883a7493a79043a9a484506c78cba62b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/e6d06ee3b3b9479a9edfeba04d4768ea, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/ff9a5320d7074362a803d3f526b64c4e, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/7c30b591713640f2b27ff576b908b27d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/1c09ac20fca443bea7b1b59729d9fd1c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/09d9e7a8dae44725a30292d6a1b8289c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/3b9d5d1d70ce4696877a06844797a93a, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/76a07bbb007242599d2489a94d6a48e5, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/d94275068961418d9ed09d91784d14cb, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/54e4a31f947948558b0e24d6abf2726e, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/d5b602c0ebc34eb0845759c6a31c957f, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/cc59d247146e403897dfa49adc1874c4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/8feb2cf6f9f44aeeaf1c88506400068c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/93aac7fff4e54b03be3125aa8e39836d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/6490b7736d5544728e0acc7024f56d7c] to archive 2024-11-09T02:23:18,500 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T02:23:18,501 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/b24c730f62a643b5a9c23acd9ba14919 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/b24c730f62a643b5a9c23acd9ba14919 2024-11-09T02:23:18,502 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/7b6a921ee6a840d186a8a30a8e280041 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/7b6a921ee6a840d186a8a30a8e280041 2024-11-09T02:23:18,503 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/27e420f2e3e6416ebd57102df880a826 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/27e420f2e3e6416ebd57102df880a826 2024-11-09T02:23:18,504 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/12ba5e23c2cc4669856c78a4ad4f48a3 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/12ba5e23c2cc4669856c78a4ad4f48a3 2024-11-09T02:23:18,505 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/6118d4a29a044ae389a43992c14cc198 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/6118d4a29a044ae389a43992c14cc198 2024-11-09T02:23:18,505 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/5c006ffcdbb547a5b9458b70114dc908 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/5c006ffcdbb547a5b9458b70114dc908 2024-11-09T02:23:18,506 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/0f8b2c0fe4424c96946161deddec14b5 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/0f8b2c0fe4424c96946161deddec14b5 2024-11-09T02:23:18,507 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/883a7493a79043a9a484506c78cba62b to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/883a7493a79043a9a484506c78cba62b 2024-11-09T02:23:18,507 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/e6d06ee3b3b9479a9edfeba04d4768ea to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/e6d06ee3b3b9479a9edfeba04d4768ea 2024-11-09T02:23:18,508 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/ff9a5320d7074362a803d3f526b64c4e to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/ff9a5320d7074362a803d3f526b64c4e 2024-11-09T02:23:18,509 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/7c30b591713640f2b27ff576b908b27d to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/7c30b591713640f2b27ff576b908b27d 2024-11-09T02:23:18,509 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/1c09ac20fca443bea7b1b59729d9fd1c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/1c09ac20fca443bea7b1b59729d9fd1c 2024-11-09T02:23:18,510 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/09d9e7a8dae44725a30292d6a1b8289c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/09d9e7a8dae44725a30292d6a1b8289c 2024-11-09T02:23:18,511 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/3b9d5d1d70ce4696877a06844797a93a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/3b9d5d1d70ce4696877a06844797a93a 2024-11-09T02:23:18,511 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/76a07bbb007242599d2489a94d6a48e5 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/76a07bbb007242599d2489a94d6a48e5 2024-11-09T02:23:18,512 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/d94275068961418d9ed09d91784d14cb to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/d94275068961418d9ed09d91784d14cb 2024-11-09T02:23:18,513 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/54e4a31f947948558b0e24d6abf2726e to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/54e4a31f947948558b0e24d6abf2726e 2024-11-09T02:23:18,514 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/d5b602c0ebc34eb0845759c6a31c957f to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/d5b602c0ebc34eb0845759c6a31c957f 2024-11-09T02:23:18,515 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/cc59d247146e403897dfa49adc1874c4 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/cc59d247146e403897dfa49adc1874c4 2024-11-09T02:23:18,516 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/8feb2cf6f9f44aeeaf1c88506400068c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/8feb2cf6f9f44aeeaf1c88506400068c 2024-11-09T02:23:18,516 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/93aac7fff4e54b03be3125aa8e39836d to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/93aac7fff4e54b03be3125aa8e39836d 2024-11-09T02:23:18,517 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/6490b7736d5544728e0acc7024f56d7c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/6490b7736d5544728e0acc7024f56d7c 2024-11-09T02:23:18,521 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/9e14c1cfa9f04886a6525830711da1e8, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/57c2ceddf3894d00b2488174830077f8, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/4f08e3604f8143829f6f025b75714715, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/ca9ca600fd274c2a8451d752187c247e, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/0f103651fbd84b97ab26ccf8f1758f13, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/fb2c76960dfc4eb6a3dafca15c11df27, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/70650045a6a149919360170d66ae4e68, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/626079bbd380479383f90cc91f625394, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/81035b55be6b4968b458f1ed0cc503ba, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/ad8295af51d54539932586a5e9ec5bbd, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/0c5e41d7034141f19ca68bde5013a259, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/855e59bef48643879202dedefe68dcdc, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/6bbab525d2874879b86e4d8b53cd42b3, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/a803131f19d540ac881fd362be5d8b1b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/a583e1e39b984d0eb4173c9b644a0db2, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/864103e6e1d940b4869f70589195d873, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/861624fbf4d84db9b52e730fb401ae03, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/798b3661a04448a4b53f5bd207b53713, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/9a834208220c498482e3f6298bff5227, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/8145c8c5f9df4b39a74e826e9193dc52, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/5f5321fd63e043d28e5ba11214e5ca85, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/3f488be2cae3488386c99f044f5afd5f] to archive 2024-11-09T02:23:18,521 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T02:23:18,523 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/9e14c1cfa9f04886a6525830711da1e8 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/9e14c1cfa9f04886a6525830711da1e8 2024-11-09T02:23:18,523 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/57c2ceddf3894d00b2488174830077f8 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/57c2ceddf3894d00b2488174830077f8 2024-11-09T02:23:18,524 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/4f08e3604f8143829f6f025b75714715 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/4f08e3604f8143829f6f025b75714715 2024-11-09T02:23:18,525 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/ca9ca600fd274c2a8451d752187c247e to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/ca9ca600fd274c2a8451d752187c247e 2024-11-09T02:23:18,526 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/0f103651fbd84b97ab26ccf8f1758f13 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/0f103651fbd84b97ab26ccf8f1758f13 2024-11-09T02:23:18,527 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/fb2c76960dfc4eb6a3dafca15c11df27 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/fb2c76960dfc4eb6a3dafca15c11df27 2024-11-09T02:23:18,527 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/70650045a6a149919360170d66ae4e68 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/70650045a6a149919360170d66ae4e68 2024-11-09T02:23:18,528 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/626079bbd380479383f90cc91f625394 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/626079bbd380479383f90cc91f625394 2024-11-09T02:23:18,529 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/81035b55be6b4968b458f1ed0cc503ba to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/81035b55be6b4968b458f1ed0cc503ba 2024-11-09T02:23:18,530 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/ad8295af51d54539932586a5e9ec5bbd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/ad8295af51d54539932586a5e9ec5bbd 2024-11-09T02:23:18,531 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/0c5e41d7034141f19ca68bde5013a259 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/0c5e41d7034141f19ca68bde5013a259 2024-11-09T02:23:18,532 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/855e59bef48643879202dedefe68dcdc to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/855e59bef48643879202dedefe68dcdc 2024-11-09T02:23:18,533 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/6bbab525d2874879b86e4d8b53cd42b3 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/6bbab525d2874879b86e4d8b53cd42b3 2024-11-09T02:23:18,534 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/a803131f19d540ac881fd362be5d8b1b to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/a803131f19d540ac881fd362be5d8b1b 2024-11-09T02:23:18,535 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/a583e1e39b984d0eb4173c9b644a0db2 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/a583e1e39b984d0eb4173c9b644a0db2 2024-11-09T02:23:18,536 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/864103e6e1d940b4869f70589195d873 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/864103e6e1d940b4869f70589195d873 2024-11-09T02:23:18,537 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/861624fbf4d84db9b52e730fb401ae03 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/861624fbf4d84db9b52e730fb401ae03 2024-11-09T02:23:18,538 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/798b3661a04448a4b53f5bd207b53713 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/798b3661a04448a4b53f5bd207b53713 2024-11-09T02:23:18,539 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/9a834208220c498482e3f6298bff5227 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/9a834208220c498482e3f6298bff5227 2024-11-09T02:23:18,540 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/8145c8c5f9df4b39a74e826e9193dc52 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/8145c8c5f9df4b39a74e826e9193dc52 2024-11-09T02:23:18,541 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/5f5321fd63e043d28e5ba11214e5ca85 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/5f5321fd63e043d28e5ba11214e5ca85 2024-11-09T02:23:18,542 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/3f488be2cae3488386c99f044f5afd5f to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/3f488be2cae3488386c99f044f5afd5f 2024-11-09T02:23:18,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/daa85faa0d01414c95e2e055538c5bab 2024-11-09T02:23:18,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/171fcde653ee4153be78c47ffabf7540 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/171fcde653ee4153be78c47ffabf7540 2024-11-09T02:23:18,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/171fcde653ee4153be78c47ffabf7540, entries=150, sequenceid=336, filesize=30.5 K 2024-11-09T02:23:18,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/689ff382417b47ea83bf1009662baf8f as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/689ff382417b47ea83bf1009662baf8f 2024-11-09T02:23:18,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/689ff382417b47ea83bf1009662baf8f, entries=150, sequenceid=336, filesize=12.0 K 2024-11-09T02:23:18,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/daa85faa0d01414c95e2e055538c5bab as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/daa85faa0d01414c95e2e055538c5bab 2024-11-09T02:23:18,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/daa85faa0d01414c95e2e055538c5bab, entries=150, sequenceid=336, filesize=12.0 K 2024-11-09T02:23:18,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=20.13 KB/20610 for a695692878ab8811a17602b58ad8f4dd in 1699ms, sequenceid=336, compaction requested=false 2024-11-09T02:23:18,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a695692878ab8811a17602b58ad8f4dd: 2024-11-09T02:23:18,784 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:18,784 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:18,784 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. after waiting 0 ms 2024-11-09T02:23:18,784 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:18,784 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(2902): Flushing a695692878ab8811a17602b58ad8f4dd 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-09T02:23:18,791 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109994e646de34b4196a54d4568edf9766f_a695692878ab8811a17602b58ad8f4dd is 50, key is test_row_0/A:col10/1731118997090/Put/seqid=0 2024-11-09T02:23:18,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742242_1418 (size=12454) 2024-11-09T02:23:19,196 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:19,207 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109994e646de34b4196a54d4568edf9766f_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109994e646de34b4196a54d4568edf9766f_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:19,208 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/aa2a7d52207b494ca1e835ea06aac107, store: [table=TestAcidGuarantees family=A region=a695692878ab8811a17602b58ad8f4dd] 2024-11-09T02:23:19,208 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/aa2a7d52207b494ca1e835ea06aac107 is 175, key is test_row_0/A:col10/1731118997090/Put/seqid=0 2024-11-09T02:23:19,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742243_1419 (size=31255) 2024-11-09T02:23:19,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T02:23:19,613 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/aa2a7d52207b494ca1e835ea06aac107 2024-11-09T02:23:19,626 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/f14bbac5f2514c0f990f2edc48c0ed99 is 50, key is test_row_0/B:col10/1731118997090/Put/seqid=0 2024-11-09T02:23:19,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742244_1420 (size=12301) 2024-11-09T02:23:20,032 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/f14bbac5f2514c0f990f2edc48c0ed99 2024-11-09T02:23:20,044 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/e622bb084e424db09c7e6b7c67b7f5a5 is 50, key is test_row_0/C:col10/1731118997090/Put/seqid=0 2024-11-09T02:23:20,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742245_1421 (size=12301) 2024-11-09T02:23:20,449 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/e622bb084e424db09c7e6b7c67b7f5a5 2024-11-09T02:23:20,459 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/A/aa2a7d52207b494ca1e835ea06aac107 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/aa2a7d52207b494ca1e835ea06aac107 2024-11-09T02:23:20,464 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/aa2a7d52207b494ca1e835ea06aac107, entries=150, sequenceid=342, filesize=30.5 K 2024-11-09T02:23:20,465 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/B/f14bbac5f2514c0f990f2edc48c0ed99 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/f14bbac5f2514c0f990f2edc48c0ed99 2024-11-09T02:23:20,470 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/f14bbac5f2514c0f990f2edc48c0ed99, entries=150, sequenceid=342, filesize=12.0 K 2024-11-09T02:23:20,470 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/.tmp/C/e622bb084e424db09c7e6b7c67b7f5a5 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/e622bb084e424db09c7e6b7c67b7f5a5 2024-11-09T02:23:20,474 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/e622bb084e424db09c7e6b7c67b7f5a5, entries=150, sequenceid=342, filesize=12.0 K 2024-11-09T02:23:20,475 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for a695692878ab8811a17602b58ad8f4dd in 1691ms, sequenceid=342, compaction requested=true 2024-11-09T02:23:20,479 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/recovered.edits/345.seqid, newMaxSeqId=345, maxSeqId=4 2024-11-09T02:23:20,480 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. 2024-11-09T02:23:20,480 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for a695692878ab8811a17602b58ad8f4dd: Waiting for close lock at 1731118997254Running coprocessor pre-close hooks at 1731118997254Disabling compacts and flushes for region at 1731118997254Disabling writes for close at 1731118998784 (+1530 ms)Obtaining lock to block concurrent updates at 1731118998784Preparing flush snapshotting stores in a695692878ab8811a17602b58ad8f4dd at 1731118998784Finished memstore snapshotting TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731118998784Flushing stores of TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd. at 1731118998785 (+1 ms)Flushing a695692878ab8811a17602b58ad8f4dd/A: creating writer at 1731118998785Flushing mob file a695692878ab8811a17602b58ad8f4dd/A: appending metadata at 1731118998790 (+5 ms)Flushing mob file a695692878ab8811a17602b58ad8f4dd/A: closing flushed file at 1731118998790Flushing a695692878ab8811a17602b58ad8f4dd/A: appending metadata at 1731118999208 (+418 ms)Flushing a695692878ab8811a17602b58ad8f4dd/A: closing flushed file at 1731118999208Flushing a695692878ab8811a17602b58ad8f4dd/B: creating writer at 1731118999623 (+415 ms)Flushing a695692878ab8811a17602b58ad8f4dd/B: appending metadata at 1731118999626 (+3 ms)Flushing a695692878ab8811a17602b58ad8f4dd/B: closing flushed file at 1731118999626Flushing a695692878ab8811a17602b58ad8f4dd/C: creating writer at 1731119000040 (+414 ms)Flushing a695692878ab8811a17602b58ad8f4dd/C: appending metadata at 1731119000043 (+3 ms)Flushing a695692878ab8811a17602b58ad8f4dd/C: closing flushed file at 1731119000043Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d90a3d8: reopening flushed file at 1731119000458 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61271fe6: reopening flushed file at 1731119000464 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47736791: reopening flushed file at 1731119000470 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for a695692878ab8811a17602b58ad8f4dd in 1691ms, sequenceid=342, compaction requested=true at 1731119000475 (+5 ms)Writing region close event to WAL at 1731119000476 (+1 ms)Running coprocessor post-close hooks at 1731119000480 (+4 ms)Closed at 1731119000480 2024-11-09T02:23:20,482 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:20,483 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=a695692878ab8811a17602b58ad8f4dd, regionState=CLOSED 2024-11-09T02:23:20,484 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 because future has completed 2024-11-09T02:23:20,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-09T02:23:20,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure a695692878ab8811a17602b58ad8f4dd, server=53f286d3af3b,46079,1731118875223 in 3.3830 sec 2024-11-09T02:23:20,489 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-09T02:23:20,489 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a695692878ab8811a17602b58ad8f4dd, UNASSIGN in 3.3880 sec 2024-11-09T02:23:20,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-09T02:23:20,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.3910 sec 2024-11-09T02:23:20,493 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731119000492"}]},"ts":"1731119000492"} 2024-11-09T02:23:20,494 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T02:23:20,494 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T02:23:20,496 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.4000 sec 2024-11-09T02:23:21,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T02:23:21,261 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:23:21,261 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T02:23:21,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:23:21,265 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:23:21,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T02:23:21,266 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:23:21,270 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:21,274 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A, FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B, FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C, FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/recovered.edits] 2024-11-09T02:23:21,280 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/02c37775bc424887a07de6c90ee3e1f9 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/02c37775bc424887a07de6c90ee3e1f9 2024-11-09T02:23:21,282 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/171fcde653ee4153be78c47ffabf7540 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/171fcde653ee4153be78c47ffabf7540 2024-11-09T02:23:21,285 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/aa2a7d52207b494ca1e835ea06aac107 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/A/aa2a7d52207b494ca1e835ea06aac107 2024-11-09T02:23:21,290 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/689ff382417b47ea83bf1009662baf8f to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/689ff382417b47ea83bf1009662baf8f 2024-11-09T02:23:21,291 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/9824d6d2f322489ea39d127c29436611 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/9824d6d2f322489ea39d127c29436611 2024-11-09T02:23:21,293 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/f14bbac5f2514c0f990f2edc48c0ed99 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/B/f14bbac5f2514c0f990f2edc48c0ed99 2024-11-09T02:23:21,295 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/28a84705ffd1465aa40b3aebbca21cb7 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/28a84705ffd1465aa40b3aebbca21cb7 2024-11-09T02:23:21,296 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/daa85faa0d01414c95e2e055538c5bab to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/daa85faa0d01414c95e2e055538c5bab 2024-11-09T02:23:21,298 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/e622bb084e424db09c7e6b7c67b7f5a5 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/C/e622bb084e424db09c7e6b7c67b7f5a5 2024-11-09T02:23:21,300 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/recovered.edits/345.seqid to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd/recovered.edits/345.seqid 2024-11-09T02:23:21,301 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:21,301 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T02:23:21,302 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T02:23:21,303 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-09T02:23:21,306 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110900d3af5843634295921ef5ddf2e8cd01_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110900d3af5843634295921ef5ddf2e8cd01_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:21,307 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091a32b92ff07f4ca09a2604f9466c52d0_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091a32b92ff07f4ca09a2604f9466c52d0_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:21,309 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109238875633abd446888936984dbdefff7_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109238875633abd446888936984dbdefff7_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:21,310 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109292a8b67ab5f4f57aac3dfa3d25a4069_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109292a8b67ab5f4f57aac3dfa3d25a4069_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:21,311 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092e9de61d81f44ad4bd7a62d3de377cc4_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092e9de61d81f44ad4bd7a62d3de377cc4_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:21,312 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110933b056a948f4412b90a9a10092109f4f_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110933b056a948f4412b90a9a10092109f4f_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:21,313 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094132f07cbe4c4f889fa448a3f264c3e9_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094132f07cbe4c4f889fa448a3f264c3e9_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:21,314 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095c7584c2b612468287294d8c5036f870_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095c7584c2b612468287294d8c5036f870_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:21,314 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110967df1f89d4004dd6a173ca9da247f46a_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110967df1f89d4004dd6a173ca9da247f46a_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:21,315 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109926bf121189f448e97b4afe727f5c9c7_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109926bf121189f448e97b4afe727f5c9c7_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:21,316 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099597b021518f4716924a63075d827bcc_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099597b021518f4716924a63075d827bcc_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:21,317 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109994e646de34b4196a54d4568edf9766f_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109994e646de34b4196a54d4568edf9766f_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:21,318 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099de24fb689014107b9d3425ff1200463_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099de24fb689014107b9d3425ff1200463_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:21,318 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099fc42cee334445c3b87ef0bc7737dca3_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099fc42cee334445c3b87ef0bc7737dca3_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:21,319 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109aa9dd900384740c988d00564a67f341d_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109aa9dd900384740c988d00564a67f341d_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:21,320 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d0dfbbbec9284eb4928546873883be7b_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d0dfbbbec9284eb4928546873883be7b_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:21,320 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d50fb56eeae948d892774aed112bf079_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d50fb56eeae948d892774aed112bf079_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:21,321 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109eee133d2158346bbb79d19e3c23c255b_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109eee133d2158346bbb79d19e3c23c255b_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:21,322 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f209c1d483444e2f9e35777c93a09068_a695692878ab8811a17602b58ad8f4dd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f209c1d483444e2f9e35777c93a09068_a695692878ab8811a17602b58ad8f4dd 2024-11-09T02:23:21,322 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T02:23:21,324 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:23:21,325 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T02:23:21,327 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T02:23:21,328 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:23:21,328 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T02:23:21,328 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731119001328"}]},"ts":"9223372036854775807"} 2024-11-09T02:23:21,329 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T02:23:21,329 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a695692878ab8811a17602b58ad8f4dd, NAME => 'TestAcidGuarantees,,1731118972935.a695692878ab8811a17602b58ad8f4dd.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T02:23:21,329 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T02:23:21,330 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731119001329"}]},"ts":"9223372036854775807"} 2024-11-09T02:23:21,331 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T02:23:21,331 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:23:21,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 70 msec 2024-11-09T02:23:21,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T02:23:21,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T02:23:21,371 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:23:21,389 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242 (was 236) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1821145813_22 at /127.0.0.1:45238 [Waiting for operation #1666] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1192768363_22 at /127.0.0.1:45240 [Waiting for operation #1641] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1192768363_22 at /127.0.0.1:45230 [Waiting for operation #1672] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1821145813_22 at /127.0.0.1:56228 [Waiting for operation #1792] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=411 (was 447), ProcessCount=11 (was 11), AvailableMemoryMB=2885 (was 2968) 2024-11-09T02:23:21,396 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=411, ProcessCount=11, AvailableMemoryMB=2885 2024-11-09T02:23:21,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T02:23:21,398 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:23:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T02:23:21,399 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:23:21,399 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:21,399 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-11-09T02:23:21,400 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:23:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-09T02:23:21,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742246_1422 (size=959) 2024-11-09T02:23:21,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-09T02:23:21,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-09T02:23:21,810 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a 2024-11-09T02:23:21,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742247_1423 (size=53) 2024-11-09T02:23:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-09T02:23:22,222 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:23:22,222 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 509cbfd6eaffef996b82cfb91370c059, disabling compactions & flushes 2024-11-09T02:23:22,222 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:22,222 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:22,222 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. after waiting 0 ms 2024-11-09T02:23:22,222 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:22,223 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:22,223 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 509cbfd6eaffef996b82cfb91370c059: Waiting for close lock at 1731119002222Disabling compacts and flushes for region at 1731119002222Disabling writes for close at 1731119002222Writing region close event to WAL at 1731119002222Closed at 1731119002222 2024-11-09T02:23:22,226 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:23:22,226 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731119002226"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731119002226"}]},"ts":"1731119002226"} 2024-11-09T02:23:22,228 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T02:23:22,229 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:23:22,230 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731119002229"}]},"ts":"1731119002229"} 2024-11-09T02:23:22,231 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T02:23:22,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=509cbfd6eaffef996b82cfb91370c059, ASSIGN}] 2024-11-09T02:23:22,233 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=509cbfd6eaffef996b82cfb91370c059, ASSIGN 2024-11-09T02:23:22,234 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=509cbfd6eaffef996b82cfb91370c059, ASSIGN; state=OFFLINE, location=53f286d3af3b,46079,1731118875223; forceNewPlan=false, retain=false 2024-11-09T02:23:22,385 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=509cbfd6eaffef996b82cfb91370c059, regionState=OPENING, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:22,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=509cbfd6eaffef996b82cfb91370c059, ASSIGN because future has completed 2024-11-09T02:23:22,389 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223}] 2024-11-09T02:23:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-09T02:23:22,550 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:22,550 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:23:22,552 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:22,552 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:23:22,553 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:22,553 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:22,555 INFO [StoreOpener-509cbfd6eaffef996b82cfb91370c059-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:22,557 INFO [StoreOpener-509cbfd6eaffef996b82cfb91370c059-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 509cbfd6eaffef996b82cfb91370c059 columnFamilyName A 2024-11-09T02:23:22,557 DEBUG [StoreOpener-509cbfd6eaffef996b82cfb91370c059-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:22,558 INFO [StoreOpener-509cbfd6eaffef996b82cfb91370c059-1 {}] regionserver.HStore(327): Store=509cbfd6eaffef996b82cfb91370c059/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:23:22,558 INFO [StoreOpener-509cbfd6eaffef996b82cfb91370c059-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:22,560 INFO [StoreOpener-509cbfd6eaffef996b82cfb91370c059-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 509cbfd6eaffef996b82cfb91370c059 columnFamilyName B 2024-11-09T02:23:22,560 DEBUG [StoreOpener-509cbfd6eaffef996b82cfb91370c059-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:22,561 INFO [StoreOpener-509cbfd6eaffef996b82cfb91370c059-1 {}] regionserver.HStore(327): Store=509cbfd6eaffef996b82cfb91370c059/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:23:22,561 INFO [StoreOpener-509cbfd6eaffef996b82cfb91370c059-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:22,563 INFO [StoreOpener-509cbfd6eaffef996b82cfb91370c059-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 509cbfd6eaffef996b82cfb91370c059 columnFamilyName C 2024-11-09T02:23:22,563 DEBUG [StoreOpener-509cbfd6eaffef996b82cfb91370c059-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:22,564 INFO [StoreOpener-509cbfd6eaffef996b82cfb91370c059-1 {}] regionserver.HStore(327): Store=509cbfd6eaffef996b82cfb91370c059/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:23:22,564 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:22,565 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:22,565 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:22,566 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:22,566 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:22,567 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T02:23:22,568 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:22,570 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:23:22,570 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened 509cbfd6eaffef996b82cfb91370c059; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65702823, jitterRate=-0.020951643586158752}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T02:23:22,571 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:22,571 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for 509cbfd6eaffef996b82cfb91370c059: Running coprocessor pre-open hook at 1731119002553Writing region info on filesystem at 1731119002553Initializing all the Stores at 1731119002554 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731119002554Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731119002554Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731119002555 (+1 ms)Cleaning up temporary data from old regions at 1731119002566 (+11 ms)Running coprocessor post-open hooks at 1731119002571 (+5 ms)Region opened successfully at 1731119002571 2024-11-09T02:23:22,572 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., pid=114, masterSystemTime=1731119002543 2024-11-09T02:23:22,574 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:22,574 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:22,575 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=509cbfd6eaffef996b82cfb91370c059, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:22,576 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 because future has completed 2024-11-09T02:23:22,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-09T02:23:22,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure 509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 in 188 msec 2024-11-09T02:23:22,581 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-09T02:23:22,581 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=509cbfd6eaffef996b82cfb91370c059, ASSIGN in 347 msec 2024-11-09T02:23:22,582 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:23:22,582 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731119002582"}]},"ts":"1731119002582"} 2024-11-09T02:23:22,583 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T02:23:22,584 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:23:22,585 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1860 sec 2024-11-09T02:23:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-09T02:23:23,551 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:23:23,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d55bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,554 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:23:23,555 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:23:23,557 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:23:23,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:23:23,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:23:23,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67f01562, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:23:23,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:23:23,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:23,560 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:23:23,561 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27a2e97d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,561 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:23:23,562 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:23:23,562 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:23:23,563 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:23:23,564 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T02:23:23,565 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 53f286d3af3b,45041,1731118874457 2024-11-09T02:23:23,565 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@972afa4 2024-11-09T02:23:23,565 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T02:23:23,566 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T02:23:23,567 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,567 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:23:23,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:23:23,568 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:23:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:23:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:23:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@558d2572, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:23:23,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:23:23,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:23,569 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:23:23,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46675063, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,571 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:23:23,571 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:23:23,571 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:23:23,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:23:23,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:23:23,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cb76a49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:23:23,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:23:23,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:23,573 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:23:23,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@752ca266, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,574 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:23:23,574 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:23:23,574 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:23:23,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:23:23,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:23:23,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@699ec346, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:23:23,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:23:23,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:23,576 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:23:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38bbc81c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,577 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,577 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:23:23,577 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:23:23,578 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:23:23,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:23:23,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:23:23,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143b65c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:23:23,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:23:23,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:23,579 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:23:23,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12aa37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc1e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,580 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:23:23,580 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:23:23,580 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:23:23,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:23:23,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:23:23,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@516bac3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:23:23,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:23:23,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:23,581 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:23:23,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a7f3af2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,582 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,582 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:23:23,582 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:23:23,583 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:23:23,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:23:23,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:23:23,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b27a765, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:23:23,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:23:23,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:23,584 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:23:23,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ee3ea8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,585 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:23:23,585 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:23:23,586 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:23:23,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:23:23,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:23:23,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@511a0f5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:23:23,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:23:23,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:23,587 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:23:23,588 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f32e6a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:23:23,589 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:23:23,589 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:23:23,589 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:23:23,589 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:23:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41f96978, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:23:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:23:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:23,590 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:23:23,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f25a2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,592 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:23:23,592 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:23:23,592 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:23:23,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:23:23,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:23:23,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19966444, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:23:23,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:23:23,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:23,593 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:23:23,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fdc03d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f5639f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,596 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:23:23,596 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:23:23,596 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:23:23,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:23:23,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:23:23,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4678e0f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:23:23,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:23:23,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:23,597 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:23:23,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad31149, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:23,598 DEBUG [Thread-1913 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:23:23,598 DEBUG [Thread-1928 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:23:23,599 DEBUG [Thread-1917 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:23:23,599 DEBUG [Thread-1924 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:23:23,599 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:23:23,599 DEBUG [Thread-1915 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:23:23,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:23:23,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:23:23,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:23:23,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:23:23,600 DEBUG [Thread-1930 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:23:23,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:23:23,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:23:23,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:23:23,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:23:23,601 DEBUG [Thread-1932 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:23:23,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:23:23,601 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:23:23,601 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:23:23,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:23:23,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:23:23,601 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:23:23,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:23:23,601 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:23:23,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-09T02:23:23,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:23:23,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:23:23,602 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:23:23,602 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:23:23,602 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:23:23,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:23:23,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:23:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:23:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:23:23,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:23:23,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:23:23,604 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:23:23,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:23:23,604 DEBUG [Thread-1921 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:23:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T02:23:23,605 DEBUG [Thread-1926 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:23:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:23:23,605 DEBUG [Thread-1919 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:23:23,605 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:23:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:23:23,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:23:23,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:23:23,606 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:23:23,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:23:23,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:23:23,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:23:23,607 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:23:23,608 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:23:23,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:23:23,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:23:23,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2] 2024-11-09T02:23:23,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:23,611 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:23:23,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 509cbfd6eaffef996b82cfb91370c059 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T02:23:23,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/bbf44f554b7a43bb824110756204e716 is 50, key is test_row_0/A:col10/1731119003610/Put/seqid=0 2024-11-09T02:23:23,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742248_1424 (size=16681) 2024-11-09T02:23:23,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119063644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:23,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:23,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119063649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:23,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:23,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:23,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119063650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:23,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:23,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119063654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:23,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119063654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T02:23:23,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T02:23:23,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:23,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:23,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:23,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:23,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:23,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119063760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:23,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:23,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119063760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:23,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:23,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:23,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119063760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:23,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:23,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119063769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:23,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:23,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119063769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:23,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T02:23:23,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119063970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119063970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119063970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:23,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T02:23:23,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:23,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:23,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:23,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:23,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:23,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119063979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119063980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:23,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:24,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/bbf44f554b7a43bb824110756204e716 2024-11-09T02:23:24,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/d46220d0d4c441e19ada3c31985e8258 is 50, key is test_row_0/B:col10/1731119003610/Put/seqid=0 2024-11-09T02:23:24,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742249_1425 (size=12001) 2024-11-09T02:23:24,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T02:23:24,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:24,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:24,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:24,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:24,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:24,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T02:23:24,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119064280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:24,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:24,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:24,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:24,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119064280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:24,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:24,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:24,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119064280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:24,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:24,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:24,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:24,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:24,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T02:23:24,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:24,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:24,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:24,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:24,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:24,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:24,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119064289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:24,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119064289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:24,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:24,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:24,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:24,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T02:23:24,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:24,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:24,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:24,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:24,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:24,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:24,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/d46220d0d4c441e19ada3c31985e8258 2024-11-09T02:23:24,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/993ae7e4ffdd404887a4d22942b2c3ab is 50, key is test_row_0/C:col10/1731119003610/Put/seqid=0 2024-11-09T02:23:24,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742250_1426 (size=12001) 2024-11-09T02:23:24,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T02:23:24,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:24,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:24,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:24,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:24,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T02:23:24,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T02:23:24,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:24,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:24,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:24,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:24,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:24,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119064790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:24,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119064790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:24,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119064790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:24,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:24,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:24,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:24,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:24,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:24,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:24,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119064800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:24,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119064800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:24,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T02:23:24,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:24,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:24,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:24,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:24,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:24,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/993ae7e4ffdd404887a4d22942b2c3ab 2024-11-09T02:23:24,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/bbf44f554b7a43bb824110756204e716 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/bbf44f554b7a43bb824110756204e716 2024-11-09T02:23:24,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/bbf44f554b7a43bb824110756204e716, entries=250, sequenceid=12, filesize=16.3 K 2024-11-09T02:23:24,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/d46220d0d4c441e19ada3c31985e8258 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/d46220d0d4c441e19ada3c31985e8258 2024-11-09T02:23:24,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/d46220d0d4c441e19ada3c31985e8258, entries=150, sequenceid=12, filesize=11.7 K 2024-11-09T02:23:24,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/993ae7e4ffdd404887a4d22942b2c3ab as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/993ae7e4ffdd404887a4d22942b2c3ab 2024-11-09T02:23:24,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/993ae7e4ffdd404887a4d22942b2c3ab, entries=150, sequenceid=12, filesize=11.7 K 2024-11-09T02:23:24,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 509cbfd6eaffef996b82cfb91370c059 in 1316ms, sequenceid=12, compaction requested=false 2024-11-09T02:23:24,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:25,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T02:23:25,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:25,056 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 509cbfd6eaffef996b82cfb91370c059 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-09T02:23:25,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/d9ec5f9f05f14dfea463effc6c181b3c is 50, key is test_row_0/A:col10/1731119003648/Put/seqid=0 2024-11-09T02:23:25,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742251_1427 (size=12001) 2024-11-09T02:23:25,467 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/d9ec5f9f05f14dfea463effc6c181b3c 2024-11-09T02:23:25,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/e9d3a51523454f1d8bdcfd56affc0374 is 50, key is test_row_0/B:col10/1731119003648/Put/seqid=0 2024-11-09T02:23:25,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742252_1428 (size=12001) 2024-11-09T02:23:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T02:23:25,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:25,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:25,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:25,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119065819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:25,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:25,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119065820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:25,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:25,824 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_2' in 509cbfd6eaffef996b82cfb91370c059 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:23:25,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119065822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:25,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:25,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:25,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:25,824 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_0' in 509cbfd6eaffef996b82cfb91370c059 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:23:25,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:25,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:25,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:25,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:25,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:25,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:25,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119065822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:25,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:25,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:25,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:25,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119065823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:25,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:25,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:25,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:25,877 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/e9d3a51523454f1d8bdcfd56affc0374 2024-11-09T02:23:25,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/612507cb595c40c29814c11c46d8d59f is 50, key is test_row_0/C:col10/1731119003648/Put/seqid=0 2024-11-09T02:23:25,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742253_1429 (size=12001) 2024-11-09T02:23:25,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:25,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119065929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:25,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:25,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:25,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:25,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:25,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119065930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:25,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:25,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:25,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:25,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:25,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119065930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:25,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:25,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:25,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:26,003 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:23:26,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119066140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:26,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:26,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119066140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:26,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:26,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119066140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:26,287 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/612507cb595c40c29814c11c46d8d59f 2024-11-09T02:23:26,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/d9ec5f9f05f14dfea463effc6c181b3c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/d9ec5f9f05f14dfea463effc6c181b3c 2024-11-09T02:23:26,297 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/d9ec5f9f05f14dfea463effc6c181b3c, entries=150, sequenceid=38, filesize=11.7 K 2024-11-09T02:23:26,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/e9d3a51523454f1d8bdcfd56affc0374 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/e9d3a51523454f1d8bdcfd56affc0374 2024-11-09T02:23:26,301 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/e9d3a51523454f1d8bdcfd56affc0374, entries=150, sequenceid=38, filesize=11.7 K 2024-11-09T02:23:26,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/612507cb595c40c29814c11c46d8d59f as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/612507cb595c40c29814c11c46d8d59f 2024-11-09T02:23:26,306 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/612507cb595c40c29814c11c46d8d59f, entries=150, sequenceid=38, filesize=11.7 K 2024-11-09T02:23:26,307 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 509cbfd6eaffef996b82cfb91370c059 in 1252ms, sequenceid=38, compaction requested=false 2024-11-09T02:23:26,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:26,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:26,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-09T02:23:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-09T02:23:26,310 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-09T02:23:26,310 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7040 sec 2024-11-09T02:23:26,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.7110 sec 2024-11-09T02:23:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:26,452 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:23:26,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 509cbfd6eaffef996b82cfb91370c059 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T02:23:26,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/a689efc4e503418283f0cd61911fca42 is 50, key is test_row_0/A:col10/1731119006451/Put/seqid=0 2024-11-09T02:23:26,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742254_1430 (size=11997) 2024-11-09T02:23:26,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:26,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119066504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119066503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:26,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:26,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:26,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:26,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:26,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:26,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:26,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119066512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:26,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:26,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:26,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:26,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119066619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:26,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:26,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119066620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:26,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119066630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:26,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:26,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:26,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:26,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119066829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:26,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:26,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:26,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119066829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:26,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:26,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:26,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:26,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:26,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119066840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:26,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:26,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:26,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:26,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/a689efc4e503418283f0cd61911fca42 2024-11-09T02:23:26,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/1b66f87dbabf48d3809dd91b6e26bf41 is 50, key is test_row_0/B:col10/1731119006451/Put/seqid=0 2024-11-09T02:23:26,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742255_1431 (size=9657) 2024-11-09T02:23:26,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/1b66f87dbabf48d3809dd91b6e26bf41 2024-11-09T02:23:26,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/f5a11b46c61e435191f21c4ee19ab2e3 is 50, key is test_row_0/C:col10/1731119006451/Put/seqid=0 2024-11-09T02:23:26,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742256_1432 (size=9657) 2024-11-09T02:23:27,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119067140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:27,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:27,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:27,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:27,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119067140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:27,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:27,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:27,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:27,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119067149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:27,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/f5a11b46c61e435191f21c4ee19ab2e3 2024-11-09T02:23:27,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/a689efc4e503418283f0cd61911fca42 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/a689efc4e503418283f0cd61911fca42 2024-11-09T02:23:27,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/a689efc4e503418283f0cd61911fca42, entries=150, sequenceid=49, filesize=11.7 K 2024-11-09T02:23:27,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/1b66f87dbabf48d3809dd91b6e26bf41 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/1b66f87dbabf48d3809dd91b6e26bf41 2024-11-09T02:23:27,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/1b66f87dbabf48d3809dd91b6e26bf41, entries=100, sequenceid=49, filesize=9.4 K 2024-11-09T02:23:27,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/f5a11b46c61e435191f21c4ee19ab2e3 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/f5a11b46c61e435191f21c4ee19ab2e3 2024-11-09T02:23:27,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/f5a11b46c61e435191f21c4ee19ab2e3, entries=100, sequenceid=49, filesize=9.4 K 2024-11-09T02:23:27,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 509cbfd6eaffef996b82cfb91370c059 in 849ms, sequenceid=49, compaction requested=true 2024-11-09T02:23:27,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:27,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 509cbfd6eaffef996b82cfb91370c059:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:23:27,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:27,301 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:27,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 509cbfd6eaffef996b82cfb91370c059:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:23:27,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:27,301 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:27,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 509cbfd6eaffef996b82cfb91370c059:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:23:27,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:27,302 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40679 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:27,302 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:27,302 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 509cbfd6eaffef996b82cfb91370c059/B is initiating minor compaction (all files) 2024-11-09T02:23:27,302 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): 509cbfd6eaffef996b82cfb91370c059/A is initiating minor compaction (all files) 2024-11-09T02:23:27,302 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 509cbfd6eaffef996b82cfb91370c059/B in TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:27,302 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 509cbfd6eaffef996b82cfb91370c059/A in TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:27,302 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/bbf44f554b7a43bb824110756204e716, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/d9ec5f9f05f14dfea463effc6c181b3c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/a689efc4e503418283f0cd61911fca42] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp, totalSize=39.7 K 2024-11-09T02:23:27,302 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/d46220d0d4c441e19ada3c31985e8258, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/e9d3a51523454f1d8bdcfd56affc0374, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/1b66f87dbabf48d3809dd91b6e26bf41] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp, totalSize=32.9 K 2024-11-09T02:23:27,302 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting d46220d0d4c441e19ada3c31985e8258, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731119003604 2024-11-09T02:23:27,302 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbf44f554b7a43bb824110756204e716, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731119003604 2024-11-09T02:23:27,302 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting e9d3a51523454f1d8bdcfd56affc0374, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731119003648 2024-11-09T02:23:27,303 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9ec5f9f05f14dfea463effc6c181b3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731119003648 2024-11-09T02:23:27,303 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b66f87dbabf48d3809dd91b6e26bf41, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731119005822 2024-11-09T02:23:27,303 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting a689efc4e503418283f0cd61911fca42, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731119005821 2024-11-09T02:23:27,317 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 509cbfd6eaffef996b82cfb91370c059#B#compaction#357 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:27,317 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 509cbfd6eaffef996b82cfb91370c059#A#compaction#358 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:27,318 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/887679cccf654a308af145725c2eea62 is 50, key is test_row_0/B:col10/1731119006451/Put/seqid=0 2024-11-09T02:23:27,318 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/53225891130646c5aeeea9b801072fff is 50, key is test_row_0/A:col10/1731119006451/Put/seqid=0 2024-11-09T02:23:27,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742258_1434 (size=12104) 2024-11-09T02:23:27,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742257_1433 (size=12104) 2024-11-09T02:23:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:27,657 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:23:27,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 509cbfd6eaffef996b82cfb91370c059 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-09T02:23:27,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/81dd713d3af44553a5e0ff1f8cecde75 is 50, key is test_row_0/A:col10/1731119006490/Put/seqid=0 2024-11-09T02:23:27,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742259_1435 (size=16681) 2024-11-09T02:23:27,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:27,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119067671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:27,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:27,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:27,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:27,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:27,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119067675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:27,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:27,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:27,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:27,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:27,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119067676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:27,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:27,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:27,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:27,728 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/887679cccf654a308af145725c2eea62 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/887679cccf654a308af145725c2eea62 2024-11-09T02:23:27,732 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 509cbfd6eaffef996b82cfb91370c059/B of 509cbfd6eaffef996b82cfb91370c059 into 887679cccf654a308af145725c2eea62(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:27,732 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:27,732 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., storeName=509cbfd6eaffef996b82cfb91370c059/B, priority=13, startTime=1731119007301; duration=0sec 2024-11-09T02:23:27,732 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:27,732 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 509cbfd6eaffef996b82cfb91370c059:B 2024-11-09T02:23:27,732 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:27,733 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:27,733 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 509cbfd6eaffef996b82cfb91370c059/C is initiating minor compaction (all files) 2024-11-09T02:23:27,733 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 509cbfd6eaffef996b82cfb91370c059/C in TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:27,733 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/993ae7e4ffdd404887a4d22942b2c3ab, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/612507cb595c40c29814c11c46d8d59f, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/f5a11b46c61e435191f21c4ee19ab2e3] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp, totalSize=32.9 K 2024-11-09T02:23:27,733 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 993ae7e4ffdd404887a4d22942b2c3ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731119003604 2024-11-09T02:23:27,733 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 612507cb595c40c29814c11c46d8d59f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731119003648 2024-11-09T02:23:27,734 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting f5a11b46c61e435191f21c4ee19ab2e3, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731119005822 2024-11-09T02:23:27,736 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/53225891130646c5aeeea9b801072fff as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/53225891130646c5aeeea9b801072fff 2024-11-09T02:23:27,743 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 509cbfd6eaffef996b82cfb91370c059/A of 509cbfd6eaffef996b82cfb91370c059 into 53225891130646c5aeeea9b801072fff(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:27,743 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:27,743 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., storeName=509cbfd6eaffef996b82cfb91370c059/A, priority=13, startTime=1731119007301; duration=0sec 2024-11-09T02:23:27,743 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:27,743 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 509cbfd6eaffef996b82cfb91370c059:A 2024-11-09T02:23:27,748 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 509cbfd6eaffef996b82cfb91370c059#C#compaction#360 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:27,748 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/ae3ec774ca2b4a0e9b6e48ded1d9c63e is 50, key is test_row_0/C:col10/1731119006451/Put/seqid=0 2024-11-09T02:23:27,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742260_1436 (size=12104) 2024-11-09T02:23:27,762 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/ae3ec774ca2b4a0e9b6e48ded1d9c63e as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/ae3ec774ca2b4a0e9b6e48ded1d9c63e 2024-11-09T02:23:27,765 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 509cbfd6eaffef996b82cfb91370c059/C of 509cbfd6eaffef996b82cfb91370c059 into ae3ec774ca2b4a0e9b6e48ded1d9c63e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:27,765 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:27,765 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., storeName=509cbfd6eaffef996b82cfb91370c059/C, priority=13, startTime=1731119007301; duration=0sec 2024-11-09T02:23:27,765 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:27,765 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 509cbfd6eaffef996b82cfb91370c059:C 2024-11-09T02:23:27,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T02:23:27,770 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:23:27,771 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:23:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-09T02:23:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T02:23:27,772 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:23:27,773 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:23:27,773 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:23:27,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119067779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:27,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:27,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:27,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:27,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119067790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:27,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119067790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:27,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119067839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:27,842 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_2' in 509cbfd6eaffef996b82cfb91370c059 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:23:27,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:27,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:27,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:27,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119067850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:27,854 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_0' in 509cbfd6eaffef996b82cfb91370c059 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:23:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T02:23:27,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-09T02:23:27,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:27,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:27,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:27,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:27,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:27,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119067990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:27,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:27,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:27,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:28,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119068000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:28,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119068000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:28,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:28,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:28,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:28,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:28,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:28,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:28,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/81dd713d3af44553a5e0ff1f8cecde75 2024-11-09T02:23:28,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/d56eb7c26e9446efb62e691efb8506f6 is 50, key is test_row_0/B:col10/1731119006490/Put/seqid=0 2024-11-09T02:23:28,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742261_1437 (size=12001) 2024-11-09T02:23:28,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/d56eb7c26e9446efb62e691efb8506f6 2024-11-09T02:23:28,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-09T02:23:28,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:28,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:28,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:28,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:28,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:28,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:28,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/96e73283ac714103be9d0902b9b900c4 is 50, key is test_row_0/C:col10/1731119006490/Put/seqid=0 2024-11-09T02:23:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T02:23:28,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742262_1438 (size=12001) 2024-11-09T02:23:28,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-09T02:23:28,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:28,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:28,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:28,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:28,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:28,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:28,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:28,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119068300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:28,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:28,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:28,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:28,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:28,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119068310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119068310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:28,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-09T02:23:28,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:28,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:28,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:28,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:28,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:28,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:28,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T02:23:28,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/96e73283ac714103be9d0902b9b900c4 2024-11-09T02:23:28,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/81dd713d3af44553a5e0ff1f8cecde75 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/81dd713d3af44553a5e0ff1f8cecde75 2024-11-09T02:23:28,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/81dd713d3af44553a5e0ff1f8cecde75, entries=250, sequenceid=76, filesize=16.3 K 2024-11-09T02:23:28,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/d56eb7c26e9446efb62e691efb8506f6 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/d56eb7c26e9446efb62e691efb8506f6 2024-11-09T02:23:28,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/d56eb7c26e9446efb62e691efb8506f6, entries=150, sequenceid=76, filesize=11.7 K 2024-11-09T02:23:28,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/96e73283ac714103be9d0902b9b900c4 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/96e73283ac714103be9d0902b9b900c4 2024-11-09T02:23:28,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/96e73283ac714103be9d0902b9b900c4, entries=150, sequenceid=76, filesize=11.7 K 2024-11-09T02:23:28,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 509cbfd6eaffef996b82cfb91370c059 in 851ms, sequenceid=76, compaction requested=false 2024-11-09T02:23:28,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:28,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-09T02:23:28,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:28,535 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 509cbfd6eaffef996b82cfb91370c059 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-09T02:23:28,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/dff49072ed464aeaa0ce3198abc6f88d is 50, key is test_row_0/A:col10/1731119007674/Put/seqid=0 2024-11-09T02:23:28,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742263_1439 (size=9657) 2024-11-09T02:23:28,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:28,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T02:23:28,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:28,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119068912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:28,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:28,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:28,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:28,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:28,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119068915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:28,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:28,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:28,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:28,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:28,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119068916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:28,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:28,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:28,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:28,942 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/dff49072ed464aeaa0ce3198abc6f88d 2024-11-09T02:23:28,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/e172b1f70990429a886e7d52d9035ff6 is 50, key is test_row_0/B:col10/1731119007674/Put/seqid=0 2024-11-09T02:23:28,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742264_1440 (size=9657) 2024-11-09T02:23:29,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:29,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119069020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:29,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:29,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:29,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:29,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119069029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:29,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:29,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:29,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119069030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:29,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:29,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:29,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:29,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:29,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119069230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:29,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:29,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:29,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:29,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119069239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:29,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119069239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:29,350 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/e172b1f70990429a886e7d52d9035ff6 2024-11-09T02:23:29,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/7bf52aefda72452e9ed0bcfa66ecf06d is 50, key is test_row_0/C:col10/1731119007674/Put/seqid=0 2024-11-09T02:23:29,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742265_1441 (size=9657) 2024-11-09T02:23:29,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119069540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:29,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:29,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:29,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:29,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119069550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:29,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119069550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:29,761 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/7bf52aefda72452e9ed0bcfa66ecf06d 2024-11-09T02:23:29,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/dff49072ed464aeaa0ce3198abc6f88d as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/dff49072ed464aeaa0ce3198abc6f88d 2024-11-09T02:23:29,766 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/dff49072ed464aeaa0ce3198abc6f88d, entries=100, sequenceid=88, filesize=9.4 K 2024-11-09T02:23:29,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/e172b1f70990429a886e7d52d9035ff6 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/e172b1f70990429a886e7d52d9035ff6 2024-11-09T02:23:29,769 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/e172b1f70990429a886e7d52d9035ff6, entries=100, sequenceid=88, filesize=9.4 K 2024-11-09T02:23:29,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/7bf52aefda72452e9ed0bcfa66ecf06d as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/7bf52aefda72452e9ed0bcfa66ecf06d 2024-11-09T02:23:29,773 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/7bf52aefda72452e9ed0bcfa66ecf06d, entries=100, sequenceid=88, filesize=9.4 K 2024-11-09T02:23:29,774 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 509cbfd6eaffef996b82cfb91370c059 in 1238ms, sequenceid=88, compaction requested=true 2024-11-09T02:23:29,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:29,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:29,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-09T02:23:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-09T02:23:29,776 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-09T02:23:29,776 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0020 sec 2024-11-09T02:23:29,777 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.0050 sec 2024-11-09T02:23:29,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T02:23:29,920 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:23:29,920 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:23:29,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-09T02:23:29,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T02:23:29,922 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:23:29,922 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:23:29,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:23:30,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T02:23:30,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:30,055 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:23:30,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 509cbfd6eaffef996b82cfb91370c059 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-09T02:23:30,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/1e42d20e84254a4eac7dd761390ad273 is 50, key is test_row_0/A:col10/1731119008905/Put/seqid=0 2024-11-09T02:23:30,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742266_1442 (size=14341) 2024-11-09T02:23:30,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/1e42d20e84254a4eac7dd761390ad273 2024-11-09T02:23:30,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/3b7c063c9b874bf6bf1489acc59c6670 is 50, key is test_row_0/B:col10/1731119008905/Put/seqid=0 2024-11-09T02:23:30,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-09T02:23:30,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:30,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:30,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:30,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:30,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:30,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:30,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119070068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:30,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:30,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:30,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:30,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119070071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:30,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:30,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119070072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:30,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:30,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:30,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742267_1443 (size=12001) 2024-11-09T02:23:30,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/3b7c063c9b874bf6bf1489acc59c6670 2024-11-09T02:23:30,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/116967be1dc8424b8b42f7b7ac76c76a is 50, key is test_row_0/C:col10/1731119008905/Put/seqid=0 2024-11-09T02:23:30,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742268_1444 (size=12001) 2024-11-09T02:23:30,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:30,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119070180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:30,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:30,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119070189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:30,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:30,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:30,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119070189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:30,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:30,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-09T02:23:30,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:30,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:30,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:30,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:30,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:30,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:30,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T02:23:30,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-09T02:23:30,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:30,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:30,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:30,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:30,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:30,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:30,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:30,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119070390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:30,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:30,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:30,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119070400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:30,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:30,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:30,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119070400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:30,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:30,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/116967be1dc8424b8b42f7b7ac76c76a 2024-11-09T02:23:30,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/1e42d20e84254a4eac7dd761390ad273 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/1e42d20e84254a4eac7dd761390ad273 2024-11-09T02:23:30,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/1e42d20e84254a4eac7dd761390ad273, entries=200, sequenceid=116, filesize=14.0 K 2024-11-09T02:23:30,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/3b7c063c9b874bf6bf1489acc59c6670 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/3b7c063c9b874bf6bf1489acc59c6670 2024-11-09T02:23:30,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/3b7c063c9b874bf6bf1489acc59c6670, entries=150, sequenceid=116, filesize=11.7 K 2024-11-09T02:23:30,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/116967be1dc8424b8b42f7b7ac76c76a as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/116967be1dc8424b8b42f7b7ac76c76a 2024-11-09T02:23:30,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/116967be1dc8424b8b42f7b7ac76c76a, entries=150, sequenceid=116, filesize=11.7 K 2024-11-09T02:23:30,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 509cbfd6eaffef996b82cfb91370c059 in 454ms, sequenceid=116, compaction requested=true 2024-11-09T02:23:30,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:30,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 509cbfd6eaffef996b82cfb91370c059:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:23:30,510 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T02:23:30,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:30,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 509cbfd6eaffef996b82cfb91370c059:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:23:30,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:30,510 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T02:23:30,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 509cbfd6eaffef996b82cfb91370c059:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:23:30,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:30,511 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T02:23:30,511 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52783 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T02:23:30,511 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 509cbfd6eaffef996b82cfb91370c059/B is initiating minor compaction (all files) 2024-11-09T02:23:30,511 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): 509cbfd6eaffef996b82cfb91370c059/A is initiating minor compaction (all files) 2024-11-09T02:23:30,511 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 509cbfd6eaffef996b82cfb91370c059/B in TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:30,511 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 509cbfd6eaffef996b82cfb91370c059/A in TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:30,511 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/887679cccf654a308af145725c2eea62, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/d56eb7c26e9446efb62e691efb8506f6, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/e172b1f70990429a886e7d52d9035ff6, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/3b7c063c9b874bf6bf1489acc59c6670] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp, totalSize=44.7 K 2024-11-09T02:23:30,511 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/53225891130646c5aeeea9b801072fff, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/81dd713d3af44553a5e0ff1f8cecde75, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/dff49072ed464aeaa0ce3198abc6f88d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/1e42d20e84254a4eac7dd761390ad273] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp, totalSize=51.5 K 2024-11-09T02:23:30,511 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53225891130646c5aeeea9b801072fff, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731119003652 2024-11-09T02:23:30,511 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 887679cccf654a308af145725c2eea62, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731119003652 2024-11-09T02:23:30,511 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81dd713d3af44553a5e0ff1f8cecde75, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731119006490 2024-11-09T02:23:30,511 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting d56eb7c26e9446efb62e691efb8506f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731119006490 2024-11-09T02:23:30,512 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting dff49072ed464aeaa0ce3198abc6f88d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731119007674 2024-11-09T02:23:30,512 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting e172b1f70990429a886e7d52d9035ff6, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731119007674 2024-11-09T02:23:30,512 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e42d20e84254a4eac7dd761390ad273, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731119008905 2024-11-09T02:23:30,512 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b7c063c9b874bf6bf1489acc59c6670, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731119008905 2024-11-09T02:23:30,519 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 509cbfd6eaffef996b82cfb91370c059#A#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:30,519 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/96427184ca8345bbaec4629b1280ff70 is 50, key is test_row_0/A:col10/1731119008905/Put/seqid=0 2024-11-09T02:23:30,522 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 509cbfd6eaffef996b82cfb91370c059#B#compaction#370 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:30,522 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/52972580832948f1867a3d327503ed0e is 50, key is test_row_0/B:col10/1731119008905/Put/seqid=0 2024-11-09T02:23:30,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742269_1445 (size=12241) 2024-11-09T02:23:30,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742270_1446 (size=12241) 2024-11-09T02:23:30,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-09T02:23:30,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:30,532 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 509cbfd6eaffef996b82cfb91370c059 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-09T02:23:30,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/dd1c6355e4994a5ca6748af354d5f61e is 50, key is test_row_1/A:col10/1731119010068/Put/seqid=0 2024-11-09T02:23:30,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742271_1447 (size=9657) 2024-11-09T02:23:30,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T02:23:30,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:30,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:30,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:30,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119070799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:30,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:30,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:30,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119070800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:30,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:30,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:30,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119070801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:30,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:30,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:30,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119070910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:30,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:30,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:30,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119070910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:30,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:30,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:30,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119070920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:30,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:30,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:30,932 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/96427184ca8345bbaec4629b1280ff70 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/96427184ca8345bbaec4629b1280ff70 2024-11-09T02:23:30,933 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/52972580832948f1867a3d327503ed0e as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/52972580832948f1867a3d327503ed0e 2024-11-09T02:23:30,935 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 509cbfd6eaffef996b82cfb91370c059/A of 509cbfd6eaffef996b82cfb91370c059 into 96427184ca8345bbaec4629b1280ff70(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:30,935 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:30,936 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., storeName=509cbfd6eaffef996b82cfb91370c059/A, priority=12, startTime=1731119010509; duration=0sec 2024-11-09T02:23:30,936 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 509cbfd6eaffef996b82cfb91370c059/B of 509cbfd6eaffef996b82cfb91370c059 into 52972580832948f1867a3d327503ed0e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:30,936 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:30,936 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:30,936 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 509cbfd6eaffef996b82cfb91370c059:A 2024-11-09T02:23:30,936 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., storeName=509cbfd6eaffef996b82cfb91370c059/B, priority=12, startTime=1731119010510; duration=0sec 2024-11-09T02:23:30,936 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T02:23:30,936 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:30,936 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 509cbfd6eaffef996b82cfb91370c059:B 2024-11-09T02:23:30,937 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T02:23:30,937 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): 509cbfd6eaffef996b82cfb91370c059/C is initiating minor compaction (all files) 2024-11-09T02:23:30,937 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 509cbfd6eaffef996b82cfb91370c059/C in TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:30,937 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/ae3ec774ca2b4a0e9b6e48ded1d9c63e, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/96e73283ac714103be9d0902b9b900c4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/7bf52aefda72452e9ed0bcfa66ecf06d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/116967be1dc8424b8b42f7b7ac76c76a] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp, totalSize=44.7 K 2024-11-09T02:23:30,937 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae3ec774ca2b4a0e9b6e48ded1d9c63e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731119003652 2024-11-09T02:23:30,937 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96e73283ac714103be9d0902b9b900c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731119006490 2024-11-09T02:23:30,937 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bf52aefda72452e9ed0bcfa66ecf06d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731119007674 2024-11-09T02:23:30,938 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 116967be1dc8424b8b42f7b7ac76c76a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731119008905 2024-11-09T02:23:30,944 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 509cbfd6eaffef996b82cfb91370c059#C#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:30,944 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/0be877724b254839b78e49136894e077 is 50, key is test_row_0/C:col10/1731119008905/Put/seqid=0 2024-11-09T02:23:30,944 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/dd1c6355e4994a5ca6748af354d5f61e 2024-11-09T02:23:30,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742272_1448 (size=12241) 2024-11-09T02:23:30,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/f4c33c0247154cb68db4403bc3a3f8c3 is 50, key is test_row_1/B:col10/1731119010068/Put/seqid=0 2024-11-09T02:23:30,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742273_1449 (size=9657) 2024-11-09T02:23:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T02:23:31,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119071120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:31,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119071120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:31,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:31,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:31,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:31,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:31,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:31,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:31,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119071130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:31,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:31,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:31,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:31,351 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/0be877724b254839b78e49136894e077 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/0be877724b254839b78e49136894e077 2024-11-09T02:23:31,353 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/f4c33c0247154cb68db4403bc3a3f8c3 2024-11-09T02:23:31,355 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 509cbfd6eaffef996b82cfb91370c059/C of 509cbfd6eaffef996b82cfb91370c059 into 0be877724b254839b78e49136894e077(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:31,355 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:31,355 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., storeName=509cbfd6eaffef996b82cfb91370c059/C, priority=12, startTime=1731119010510; duration=0sec 2024-11-09T02:23:31,355 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:31,355 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 509cbfd6eaffef996b82cfb91370c059:C 2024-11-09T02:23:31,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/349ac1735b0c4279aa201b480a2ada71 is 50, key is test_row_1/C:col10/1731119010068/Put/seqid=0 2024-11-09T02:23:31,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742274_1450 (size=9657) 2024-11-09T02:23:31,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119071430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:31,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:31,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:31,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119071430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:31,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:31,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:31,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:31,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:31,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119071439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:31,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:31,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:31,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:31,762 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/349ac1735b0c4279aa201b480a2ada71 2024-11-09T02:23:31,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/dd1c6355e4994a5ca6748af354d5f61e as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/dd1c6355e4994a5ca6748af354d5f61e 2024-11-09T02:23:31,768 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/dd1c6355e4994a5ca6748af354d5f61e, entries=100, sequenceid=124, filesize=9.4 K 2024-11-09T02:23:31,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/f4c33c0247154cb68db4403bc3a3f8c3 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/f4c33c0247154cb68db4403bc3a3f8c3 2024-11-09T02:23:31,772 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/f4c33c0247154cb68db4403bc3a3f8c3, entries=100, sequenceid=124, filesize=9.4 K 2024-11-09T02:23:31,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/349ac1735b0c4279aa201b480a2ada71 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/349ac1735b0c4279aa201b480a2ada71 2024-11-09T02:23:31,775 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/349ac1735b0c4279aa201b480a2ada71, entries=100, sequenceid=124, filesize=9.4 K 2024-11-09T02:23:31,776 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 509cbfd6eaffef996b82cfb91370c059 in 1244ms, sequenceid=124, compaction requested=false 2024-11-09T02:23:31,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:31,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:31,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-09T02:23:31,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-09T02:23:31,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-09T02:23:31,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8550 sec 2024-11-09T02:23:31,780 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.8590 sec 2024-11-09T02:23:31,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:31,874 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:23:31,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 509cbfd6eaffef996b82cfb91370c059 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-09T02:23:31,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/5b2c20e368814c6287232d24e9b97545 is 50, key is test_row_0/A:col10/1731119011873/Put/seqid=0 2024-11-09T02:23:31,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742275_1451 (size=14541) 2024-11-09T02:23:31,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:31,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119071885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:31,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:31,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:31,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:31,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:31,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119071891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:31,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:31,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:31,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119071940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:31,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:31,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119071940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:31,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:31,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119071950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:31,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:31,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:31,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:32,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:32,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119072000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:32,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:32,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:32,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:32,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119072000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:32,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:32,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:32,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:32,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:32,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T02:23:32,070 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:23:32,070 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:23:32,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-09T02:23:32,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T02:23:32,072 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:23:32,072 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:23:32,072 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:23:32,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T02:23:32,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:32,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119072210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:32,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:32,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:32,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:32,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:32,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119072210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:32,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:32,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:32,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:32,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T02:23:32,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:32,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:32,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:32,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:32,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:32,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:32,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/5b2c20e368814c6287232d24e9b97545 2024-11-09T02:23:32,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/06b32d9fb1ad4209b2b74be29a3c7752 is 50, key is test_row_0/B:col10/1731119011873/Put/seqid=0 2024-11-09T02:23:32,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742276_1452 (size=12151) 2024-11-09T02:23:32,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T02:23:32,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:32,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:32,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:32,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:32,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:32,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:32,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T02:23:32,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:32,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119072520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:32,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:32,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:32,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:32,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:32,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119072520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:32,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:32,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:32,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:32,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T02:23:32,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:32,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:32,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:32,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:32,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:32,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:32,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T02:23:32,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:32,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:32,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:32,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:32,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:32,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:32,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/06b32d9fb1ad4209b2b74be29a3c7752 2024-11-09T02:23:32,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T02:23:32,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/788a35eddbf54d84a0bb9a36819923fa is 50, key is test_row_0/C:col10/1731119011873/Put/seqid=0 2024-11-09T02:23:32,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742277_1453 (size=12151) 2024-11-09T02:23:32,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T02:23:32,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:32,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:32,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:32,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:32,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:32,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:32,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:32,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119072950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:32,951 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_0' in 509cbfd6eaffef996b82cfb91370c059 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:23:32,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:32,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:32,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:32,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119072960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:32,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119072960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:32,964 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_2' in 509cbfd6eaffef996b82cfb91370c059 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:23:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:32,964 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_0' in 509cbfd6eaffef996b82cfb91370c059 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:23:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:32,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T02:23:32,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:32,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:32,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:32,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:32,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:33,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119073030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:33,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:33,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:33,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:33,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119073030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:33,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/788a35eddbf54d84a0bb9a36819923fa 2024-11-09T02:23:33,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/5b2c20e368814c6287232d24e9b97545 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/5b2c20e368814c6287232d24e9b97545 2024-11-09T02:23:33,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/5b2c20e368814c6287232d24e9b97545, entries=200, sequenceid=156, filesize=14.2 K 2024-11-09T02:23:33,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/06b32d9fb1ad4209b2b74be29a3c7752 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/06b32d9fb1ad4209b2b74be29a3c7752 2024-11-09T02:23:33,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/06b32d9fb1ad4209b2b74be29a3c7752, entries=150, sequenceid=156, filesize=11.9 K 2024-11-09T02:23:33,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/788a35eddbf54d84a0bb9a36819923fa as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/788a35eddbf54d84a0bb9a36819923fa 2024-11-09T02:23:33,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/788a35eddbf54d84a0bb9a36819923fa, entries=150, sequenceid=156, filesize=11.9 K 2024-11-09T02:23:33,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 509cbfd6eaffef996b82cfb91370c059 in 1245ms, sequenceid=156, compaction requested=true 2024-11-09T02:23:33,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:33,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 509cbfd6eaffef996b82cfb91370c059:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:23:33,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:33,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 509cbfd6eaffef996b82cfb91370c059:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:23:33,119 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:33,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:33,119 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:33,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 509cbfd6eaffef996b82cfb91370c059:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:23:33,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:33,120 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:33,120 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 509cbfd6eaffef996b82cfb91370c059/B is initiating minor compaction (all files) 2024-11-09T02:23:33,120 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 509cbfd6eaffef996b82cfb91370c059/B in TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:33,120 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/52972580832948f1867a3d327503ed0e, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/f4c33c0247154cb68db4403bc3a3f8c3, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/06b32d9fb1ad4209b2b74be29a3c7752] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp, totalSize=33.3 K 2024-11-09T02:23:33,120 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36439 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:33,120 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): 509cbfd6eaffef996b82cfb91370c059/A is initiating minor compaction (all files) 2024-11-09T02:23:33,120 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 509cbfd6eaffef996b82cfb91370c059/A in TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:33,120 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/96427184ca8345bbaec4629b1280ff70, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/dd1c6355e4994a5ca6748af354d5f61e, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/5b2c20e368814c6287232d24e9b97545] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp, totalSize=35.6 K 2024-11-09T02:23:33,121 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 52972580832948f1867a3d327503ed0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731119008905 2024-11-09T02:23:33,121 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96427184ca8345bbaec4629b1280ff70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731119008905 2024-11-09T02:23:33,121 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting f4c33c0247154cb68db4403bc3a3f8c3, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731119010068 2024-11-09T02:23:33,121 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd1c6355e4994a5ca6748af354d5f61e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731119010068 2024-11-09T02:23:33,121 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 06b32d9fb1ad4209b2b74be29a3c7752, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731119010799 2024-11-09T02:23:33,121 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b2c20e368814c6287232d24e9b97545, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731119010798 2024-11-09T02:23:33,127 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 509cbfd6eaffef996b82cfb91370c059#B#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:33,127 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/5d97933de84d4cabb4fb21a49da8cf17 is 50, key is test_row_0/B:col10/1731119011873/Put/seqid=0 2024-11-09T02:23:33,133 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 509cbfd6eaffef996b82cfb91370c059#A#compaction#379 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:33,134 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/b4592cc9cba44276a711bd9dc2ca85ba is 50, key is test_row_0/A:col10/1731119011873/Put/seqid=0 2024-11-09T02:23:33,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T02:23:33,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:33,140 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 509cbfd6eaffef996b82cfb91370c059 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-09T02:23:33,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/8b8654ddd61042ee96e2073f0e7c8e4c is 50, key is test_row_0/A:col10/1731119011881/Put/seqid=0 2024-11-09T02:23:33,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742278_1454 (size=12493) 2024-11-09T02:23:33,157 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/5d97933de84d4cabb4fb21a49da8cf17 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/5d97933de84d4cabb4fb21a49da8cf17 2024-11-09T02:23:33,162 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 509cbfd6eaffef996b82cfb91370c059/B of 509cbfd6eaffef996b82cfb91370c059 into 5d97933de84d4cabb4fb21a49da8cf17(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:33,163 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:33,163 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., storeName=509cbfd6eaffef996b82cfb91370c059/B, priority=13, startTime=1731119013119; duration=0sec 2024-11-09T02:23:33,163 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:33,163 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 509cbfd6eaffef996b82cfb91370c059:B 2024-11-09T02:23:33,163 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:33,163 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:33,163 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 509cbfd6eaffef996b82cfb91370c059/C is initiating minor compaction (all files) 2024-11-09T02:23:33,163 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 509cbfd6eaffef996b82cfb91370c059/C in TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:33,163 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/0be877724b254839b78e49136894e077, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/349ac1735b0c4279aa201b480a2ada71, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/788a35eddbf54d84a0bb9a36819923fa] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp, totalSize=33.3 K 2024-11-09T02:23:33,164 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 0be877724b254839b78e49136894e077, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731119008905 2024-11-09T02:23:33,164 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 349ac1735b0c4279aa201b480a2ada71, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731119010068 2024-11-09T02:23:33,165 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 788a35eddbf54d84a0bb9a36819923fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731119010799 2024-11-09T02:23:33,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742279_1455 (size=12493) 2024-11-09T02:23:33,178 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/b4592cc9cba44276a711bd9dc2ca85ba as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/b4592cc9cba44276a711bd9dc2ca85ba 2024-11-09T02:23:33,182 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 509cbfd6eaffef996b82cfb91370c059#C#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:33,182 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/e764533927af408a8271f8e0936fbb44 is 50, key is test_row_0/C:col10/1731119011873/Put/seqid=0 2024-11-09T02:23:33,184 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 509cbfd6eaffef996b82cfb91370c059/A of 509cbfd6eaffef996b82cfb91370c059 into b4592cc9cba44276a711bd9dc2ca85ba(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:33,184 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:33,184 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., storeName=509cbfd6eaffef996b82cfb91370c059/A, priority=13, startTime=1731119013119; duration=0sec 2024-11-09T02:23:33,184 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:33,184 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 509cbfd6eaffef996b82cfb91370c059:A 2024-11-09T02:23:33,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742280_1456 (size=12151) 2024-11-09T02:23:33,189 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/8b8654ddd61042ee96e2073f0e7c8e4c 2024-11-09T02:23:33,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/d50e4bf8615a40f7800a2dc027ce8e36 is 50, key is test_row_0/B:col10/1731119011881/Put/seqid=0 2024-11-09T02:23:33,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742282_1458 (size=12151) 2024-11-09T02:23:33,202 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/d50e4bf8615a40f7800a2dc027ce8e36 2024-11-09T02:23:33,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742281_1457 (size=12493) 2024-11-09T02:23:33,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/b95b6474144946248b4181023b77ed16 is 50, key is test_row_0/C:col10/1731119011881/Put/seqid=0 2024-11-09T02:23:33,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T02:23:33,214 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/e764533927af408a8271f8e0936fbb44 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/e764533927af408a8271f8e0936fbb44 2024-11-09T02:23:33,222 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 509cbfd6eaffef996b82cfb91370c059/C of 509cbfd6eaffef996b82cfb91370c059 into e764533927af408a8271f8e0936fbb44(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:33,222 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:33,222 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., storeName=509cbfd6eaffef996b82cfb91370c059/C, priority=13, startTime=1731119013119; duration=0sec 2024-11-09T02:23:33,222 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:33,222 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 509cbfd6eaffef996b82cfb91370c059:C 2024-11-09T02:23:33,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742283_1459 (size=12151) 2024-11-09T02:23:33,625 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/b95b6474144946248b4181023b77ed16 2024-11-09T02:23:33,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/8b8654ddd61042ee96e2073f0e7c8e4c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/8b8654ddd61042ee96e2073f0e7c8e4c 2024-11-09T02:23:33,632 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/8b8654ddd61042ee96e2073f0e7c8e4c, entries=150, sequenceid=163, filesize=11.9 K 2024-11-09T02:23:33,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/d50e4bf8615a40f7800a2dc027ce8e36 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/d50e4bf8615a40f7800a2dc027ce8e36 2024-11-09T02:23:33,636 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/d50e4bf8615a40f7800a2dc027ce8e36, entries=150, sequenceid=163, filesize=11.9 K 2024-11-09T02:23:33,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/b95b6474144946248b4181023b77ed16 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/b95b6474144946248b4181023b77ed16 2024-11-09T02:23:33,640 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/b95b6474144946248b4181023b77ed16, entries=150, sequenceid=163, filesize=11.9 K 2024-11-09T02:23:33,641 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 509cbfd6eaffef996b82cfb91370c059 in 501ms, sequenceid=163, compaction requested=false 2024-11-09T02:23:33,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:33,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:33,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-09T02:23:33,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-09T02:23:33,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-09T02:23:33,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5700 sec 2024-11-09T02:23:33,648 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.5740 sec 2024-11-09T02:23:34,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:34,107 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:23:34,107 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 509cbfd6eaffef996b82cfb91370c059 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T02:23:34,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/20d0904feb8e4c66a3ead86fefaeab72 is 50, key is test_row_0/A:col10/1731119014106/Put/seqid=0 2024-11-09T02:23:34,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742284_1460 (size=16931) 2024-11-09T02:23:34,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:34,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119074179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:34,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:34,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:34,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119074184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:34,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:34,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:34,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:34,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:34,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:34,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T02:23:34,220 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:23:34,220 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:23:34,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-09T02:23:34,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T02:23:34,221 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:23:34,222 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:23:34,222 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:23:34,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:34,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119074300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:34,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:34,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:34,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:34,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119074300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:34,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:34,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:34,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:34,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:34,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T02:23:34,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T02:23:34,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:34,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:34,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:34,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:34,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:34,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:34,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:34,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119074510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:34,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:34,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:34,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119074510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:34,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:34,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:34,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/20d0904feb8e4c66a3ead86fefaeab72 2024-11-09T02:23:34,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:34,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:34,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:34,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/6bc9001c99f94f0ab9ec8f264f5138f0 is 50, key is test_row_0/B:col10/1731119014106/Put/seqid=0 2024-11-09T02:23:34,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742285_1461 (size=12151) 2024-11-09T02:23:34,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T02:23:34,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:34,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:34,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:34,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:34,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:34,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:34,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T02:23:34,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T02:23:34,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:34,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:34,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:34,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:34,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:34,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T02:23:34,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:34,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:34,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:34,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:34,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:34,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:34,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119074829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:34,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:34,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:34,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:34,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:34,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119074830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:34,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:34,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:34,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T02:23:34,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/6bc9001c99f94f0ab9ec8f264f5138f0 2024-11-09T02:23:34,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/076a511eafb14bf995a2e6bb74a93463 is 50, key is test_row_0/C:col10/1731119014106/Put/seqid=0 2024-11-09T02:23:34,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742286_1462 (size=12151) 2024-11-09T02:23:34,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119074970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:34,979 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_0' in 509cbfd6eaffef996b82cfb91370c059 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:23:34,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:34,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:34,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:34,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T02:23:34,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:34,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:34,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:34,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:34,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:34,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:34,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119074980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:34,991 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_2' in 509cbfd6eaffef996b82cfb91370c059 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:23:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:34,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:34,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119074989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:34,996 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_0' in 509cbfd6eaffef996b82cfb91370c059 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:23:34,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:34,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:34,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:35,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T02:23:35,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:35,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:35,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:35,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:35,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:35,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T02:23:35,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:35,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:35,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:35,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:35,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:35,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:35,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/076a511eafb14bf995a2e6bb74a93463 2024-11-09T02:23:35,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/20d0904feb8e4c66a3ead86fefaeab72 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/20d0904feb8e4c66a3ead86fefaeab72 2024-11-09T02:23:35,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/20d0904feb8e4c66a3ead86fefaeab72, entries=250, sequenceid=177, filesize=16.5 K 2024-11-09T02:23:35,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/6bc9001c99f94f0ab9ec8f264f5138f0 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/6bc9001c99f94f0ab9ec8f264f5138f0 2024-11-09T02:23:35,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/6bc9001c99f94f0ab9ec8f264f5138f0, entries=150, sequenceid=177, filesize=11.9 K 2024-11-09T02:23:35,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:35,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119075340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:35,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:35,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:35,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:35,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119075340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:35,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/076a511eafb14bf995a2e6bb74a93463 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/076a511eafb14bf995a2e6bb74a93463 2024-11-09T02:23:35,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:35,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:35,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:35,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:35,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/076a511eafb14bf995a2e6bb74a93463, entries=150, sequenceid=177, filesize=11.9 K 2024-11-09T02:23:35,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 509cbfd6eaffef996b82cfb91370c059 in 1243ms, sequenceid=177, compaction requested=true 2024-11-09T02:23:35,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:35,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 509cbfd6eaffef996b82cfb91370c059:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:23:35,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:35,350 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:35,350 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:35,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 509cbfd6eaffef996b82cfb91370c059:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:23:35,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:35,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 509cbfd6eaffef996b82cfb91370c059:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:23:35,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:35,351 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:35,351 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 509cbfd6eaffef996b82cfb91370c059/B is initiating minor compaction (all files) 2024-11-09T02:23:35,351 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 509cbfd6eaffef996b82cfb91370c059/B in TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:35,351 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/5d97933de84d4cabb4fb21a49da8cf17, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/d50e4bf8615a40f7800a2dc027ce8e36, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/6bc9001c99f94f0ab9ec8f264f5138f0] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp, totalSize=35.9 K 2024-11-09T02:23:35,351 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:35,351 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): 509cbfd6eaffef996b82cfb91370c059/A is initiating minor compaction (all files) 2024-11-09T02:23:35,351 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 509cbfd6eaffef996b82cfb91370c059/A in TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:35,351 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/b4592cc9cba44276a711bd9dc2ca85ba, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/8b8654ddd61042ee96e2073f0e7c8e4c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/20d0904feb8e4c66a3ead86fefaeab72] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp, totalSize=40.6 K 2024-11-09T02:23:35,351 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4592cc9cba44276a711bd9dc2ca85ba, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731119010799 2024-11-09T02:23:35,351 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d97933de84d4cabb4fb21a49da8cf17, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731119010799 2024-11-09T02:23:35,352 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting d50e4bf8615a40f7800a2dc027ce8e36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731119011881 2024-11-09T02:23:35,352 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b8654ddd61042ee96e2073f0e7c8e4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731119011881 2024-11-09T02:23:35,352 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bc9001c99f94f0ab9ec8f264f5138f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731119014101 2024-11-09T02:23:35,352 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20d0904feb8e4c66a3ead86fefaeab72, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731119014099 2024-11-09T02:23:35,357 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 509cbfd6eaffef996b82cfb91370c059#A#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:35,358 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 509cbfd6eaffef996b82cfb91370c059#B#compaction#388 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:35,358 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/7bddb510e61a48fabd73c435dc8a03f4 is 50, key is test_row_0/A:col10/1731119014106/Put/seqid=0 2024-11-09T02:23:35,358 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/322ea8d3c0ae46bd9ca4c1b4916855ac is 50, key is test_row_0/B:col10/1731119014106/Put/seqid=0 2024-11-09T02:23:35,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T02:23:35,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742288_1464 (size=12595) 2024-11-09T02:23:35,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742287_1463 (size=12595) 2024-11-09T02:23:35,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T02:23:35,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:35,443 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 509cbfd6eaffef996b82cfb91370c059 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T02:23:35,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/2534285fdd74423d820f12f4607c22ba is 50, key is test_row_0/A:col10/1731119014176/Put/seqid=0 2024-11-09T02:23:35,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742289_1465 (size=12151) 2024-11-09T02:23:35,772 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/7bddb510e61a48fabd73c435dc8a03f4 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/7bddb510e61a48fabd73c435dc8a03f4 2024-11-09T02:23:35,774 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/322ea8d3c0ae46bd9ca4c1b4916855ac as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/322ea8d3c0ae46bd9ca4c1b4916855ac 2024-11-09T02:23:35,776 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 509cbfd6eaffef996b82cfb91370c059/A of 509cbfd6eaffef996b82cfb91370c059 into 7bddb510e61a48fabd73c435dc8a03f4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:35,776 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:35,776 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., storeName=509cbfd6eaffef996b82cfb91370c059/A, priority=13, startTime=1731119015350; duration=0sec 2024-11-09T02:23:35,776 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:35,776 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 509cbfd6eaffef996b82cfb91370c059:A 2024-11-09T02:23:35,776 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:35,777 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 509cbfd6eaffef996b82cfb91370c059/B of 509cbfd6eaffef996b82cfb91370c059 into 322ea8d3c0ae46bd9ca4c1b4916855ac(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:35,777 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:35,777 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:35,777 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., storeName=509cbfd6eaffef996b82cfb91370c059/B, priority=13, startTime=1731119015350; duration=0sec 2024-11-09T02:23:35,777 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): 509cbfd6eaffef996b82cfb91370c059/C is initiating minor compaction (all files) 2024-11-09T02:23:35,777 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:35,777 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 509cbfd6eaffef996b82cfb91370c059/C in TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:35,777 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 509cbfd6eaffef996b82cfb91370c059:B 2024-11-09T02:23:35,777 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/e764533927af408a8271f8e0936fbb44, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/b95b6474144946248b4181023b77ed16, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/076a511eafb14bf995a2e6bb74a93463] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp, totalSize=35.9 K 2024-11-09T02:23:35,777 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting e764533927af408a8271f8e0936fbb44, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731119010799 2024-11-09T02:23:35,777 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting b95b6474144946248b4181023b77ed16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731119011881 2024-11-09T02:23:35,778 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 076a511eafb14bf995a2e6bb74a93463, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731119014101 2024-11-09T02:23:35,783 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 509cbfd6eaffef996b82cfb91370c059#C#compaction#390 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:35,783 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/92268d3e35494456a51655f8d1e612e7 is 50, key is test_row_0/C:col10/1731119014106/Put/seqid=0 2024-11-09T02:23:35,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742290_1466 (size=12595) 2024-11-09T02:23:35,850 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/2534285fdd74423d820f12f4607c22ba 2024-11-09T02:23:35,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/d971da954c114578bd6cad02619b27aa is 50, key is test_row_0/B:col10/1731119014176/Put/seqid=0 2024-11-09T02:23:35,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742291_1467 (size=12151) 2024-11-09T02:23:36,189 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/92268d3e35494456a51655f8d1e612e7 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/92268d3e35494456a51655f8d1e612e7 2024-11-09T02:23:36,193 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 509cbfd6eaffef996b82cfb91370c059/C of 509cbfd6eaffef996b82cfb91370c059 into 92268d3e35494456a51655f8d1e612e7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:36,193 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:36,193 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., storeName=509cbfd6eaffef996b82cfb91370c059/C, priority=13, startTime=1731119015350; duration=0sec 2024-11-09T02:23:36,193 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:36,193 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 509cbfd6eaffef996b82cfb91370c059:C 2024-11-09T02:23:36,259 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/d971da954c114578bd6cad02619b27aa 2024-11-09T02:23:36,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/3e58c650b0e144b991233dcc2958be2c is 50, key is test_row_0/C:col10/1731119014176/Put/seqid=0 2024-11-09T02:23:36,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742292_1468 (size=12151) 2024-11-09T02:23:36,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:36,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T02:23:36,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119076371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:36,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:36,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119076372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:36,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:36,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:36,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:36,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119076479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:36,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:36,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:36,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:36,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119076490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:36,675 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/3e58c650b0e144b991233dcc2958be2c 2024-11-09T02:23:36,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/2534285fdd74423d820f12f4607c22ba as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/2534285fdd74423d820f12f4607c22ba 2024-11-09T02:23:36,681 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/2534285fdd74423d820f12f4607c22ba, entries=150, sequenceid=202, filesize=11.9 K 2024-11-09T02:23:36,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/d971da954c114578bd6cad02619b27aa as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/d971da954c114578bd6cad02619b27aa 2024-11-09T02:23:36,685 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/d971da954c114578bd6cad02619b27aa, entries=150, sequenceid=202, filesize=11.9 K 2024-11-09T02:23:36,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/3e58c650b0e144b991233dcc2958be2c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/3e58c650b0e144b991233dcc2958be2c 2024-11-09T02:23:36,688 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/3e58c650b0e144b991233dcc2958be2c, entries=150, sequenceid=202, filesize=11.9 K 2024-11-09T02:23:36,688 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 509cbfd6eaffef996b82cfb91370c059 in 1246ms, sequenceid=202, compaction requested=false 2024-11-09T02:23:36,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:36,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:36,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-09T02:23:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-09T02:23:36,691 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-09T02:23:36,691 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4680 sec 2024-11-09T02:23:36,692 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.4710 sec 2024-11-09T02:23:36,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:36,696 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:23:36,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 509cbfd6eaffef996b82cfb91370c059 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T02:23:36,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/b4fa37fbb1ea46b084552e391772fcaa is 50, key is test_row_0/A:col10/1731119016371/Put/seqid=0 2024-11-09T02:23:36,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742293_1469 (size=14541) 2024-11-09T02:23:36,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119076768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:36,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:36,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:36,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:36,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119076771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:36,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:36,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:36,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:36,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:36,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119076880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:36,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:36,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119076880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:36,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:36,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:36,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:36,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:36,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:36,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:37,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/b4fa37fbb1ea46b084552e391772fcaa 2024-11-09T02:23:37,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:37,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119077100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:37,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:37,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:37,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119077100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:37,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:37,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:37,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:37,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:37,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:37,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/5e0698052ede45148a21bc6f32508d2d is 50, key is test_row_0/B:col10/1731119016371/Put/seqid=0 2024-11-09T02:23:37,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742294_1470 (size=12151) 2024-11-09T02:23:37,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:37,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119077410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:37,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:37,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119077410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:37,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:37,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:37,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:37,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:37,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:37,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:37,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/5e0698052ede45148a21bc6f32508d2d 2024-11-09T02:23:37,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/baceb1e3d3b5490cb8da8da4af75a0b4 is 50, key is test_row_0/C:col10/1731119016371/Put/seqid=0 2024-11-09T02:23:37,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742295_1471 (size=12151) 2024-11-09T02:23:37,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/baceb1e3d3b5490cb8da8da4af75a0b4 2024-11-09T02:23:37,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/b4fa37fbb1ea46b084552e391772fcaa as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/b4fa37fbb1ea46b084552e391772fcaa 2024-11-09T02:23:37,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/b4fa37fbb1ea46b084552e391772fcaa, entries=200, sequenceid=217, filesize=14.2 K 2024-11-09T02:23:37,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/5e0698052ede45148a21bc6f32508d2d as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/5e0698052ede45148a21bc6f32508d2d 2024-11-09T02:23:37,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/5e0698052ede45148a21bc6f32508d2d, entries=150, sequenceid=217, filesize=11.9 K 2024-11-09T02:23:37,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/baceb1e3d3b5490cb8da8da4af75a0b4 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/baceb1e3d3b5490cb8da8da4af75a0b4 2024-11-09T02:23:37,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119077930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:37,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:37,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:37,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:37,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119077930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:37,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:37,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:37,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:37,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/baceb1e3d3b5490cb8da8da4af75a0b4, entries=150, sequenceid=217, filesize=11.9 K 2024-11-09T02:23:37,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 509cbfd6eaffef996b82cfb91370c059 in 1241ms, sequenceid=217, compaction requested=true 2024-11-09T02:23:37,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:37,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 509cbfd6eaffef996b82cfb91370c059:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:23:37,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:37,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 509cbfd6eaffef996b82cfb91370c059:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:23:37,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:37,938 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:37,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 509cbfd6eaffef996b82cfb91370c059:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:23:37,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:37,938 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:37,938 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:37,938 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:37,938 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 509cbfd6eaffef996b82cfb91370c059/B is initiating minor compaction (all files) 2024-11-09T02:23:37,938 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): 509cbfd6eaffef996b82cfb91370c059/A is initiating minor compaction (all files) 2024-11-09T02:23:37,938 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 509cbfd6eaffef996b82cfb91370c059/B in TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:37,938 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 509cbfd6eaffef996b82cfb91370c059/A in TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:37,939 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/322ea8d3c0ae46bd9ca4c1b4916855ac, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/d971da954c114578bd6cad02619b27aa, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/5e0698052ede45148a21bc6f32508d2d] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp, totalSize=36.0 K 2024-11-09T02:23:37,939 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/7bddb510e61a48fabd73c435dc8a03f4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/2534285fdd74423d820f12f4607c22ba, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/b4fa37fbb1ea46b084552e391772fcaa] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp, totalSize=38.4 K 2024-11-09T02:23:37,939 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bddb510e61a48fabd73c435dc8a03f4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731119014101 2024-11-09T02:23:37,939 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 322ea8d3c0ae46bd9ca4c1b4916855ac, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731119014101 2024-11-09T02:23:37,939 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting d971da954c114578bd6cad02619b27aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731119014163 2024-11-09T02:23:37,939 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2534285fdd74423d820f12f4607c22ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731119014163 2024-11-09T02:23:37,939 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e0698052ede45148a21bc6f32508d2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731119016362 2024-11-09T02:23:37,939 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4fa37fbb1ea46b084552e391772fcaa, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731119016362 2024-11-09T02:23:37,947 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 509cbfd6eaffef996b82cfb91370c059#B#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:37,948 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/e420811a04a5437bb723e2085d473cb4 is 50, key is test_row_0/B:col10/1731119016371/Put/seqid=0 2024-11-09T02:23:37,948 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 509cbfd6eaffef996b82cfb91370c059#A#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:37,948 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/55ae7943f9e34f3698732f2afccfd540 is 50, key is test_row_0/A:col10/1731119016371/Put/seqid=0 2024-11-09T02:23:37,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742296_1472 (size=12697) 2024-11-09T02:23:37,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742297_1473 (size=12697) 2024-11-09T02:23:37,966 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/55ae7943f9e34f3698732f2afccfd540 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/55ae7943f9e34f3698732f2afccfd540 2024-11-09T02:23:37,969 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 509cbfd6eaffef996b82cfb91370c059/A of 509cbfd6eaffef996b82cfb91370c059 into 55ae7943f9e34f3698732f2afccfd540(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:37,969 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:37,969 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., storeName=509cbfd6eaffef996b82cfb91370c059/A, priority=13, startTime=1731119017938; duration=0sec 2024-11-09T02:23:37,969 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:37,969 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 509cbfd6eaffef996b82cfb91370c059:A 2024-11-09T02:23:37,969 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:37,972 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:37,972 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): 509cbfd6eaffef996b82cfb91370c059/C is initiating minor compaction (all files) 2024-11-09T02:23:37,972 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 509cbfd6eaffef996b82cfb91370c059/C in TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:37,972 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/92268d3e35494456a51655f8d1e612e7, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/3e58c650b0e144b991233dcc2958be2c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/baceb1e3d3b5490cb8da8da4af75a0b4] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp, totalSize=36.0 K 2024-11-09T02:23:37,972 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92268d3e35494456a51655f8d1e612e7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731119014101 2024-11-09T02:23:37,973 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e58c650b0e144b991233dcc2958be2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731119014163 2024-11-09T02:23:37,973 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting baceb1e3d3b5490cb8da8da4af75a0b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731119016362 2024-11-09T02:23:37,978 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 509cbfd6eaffef996b82cfb91370c059#C#compaction#398 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:37,978 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/325ed754411d450cbe1e7ae29e807520 is 50, key is test_row_0/C:col10/1731119016371/Put/seqid=0 2024-11-09T02:23:37,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742298_1474 (size=12697) 2024-11-09T02:23:38,363 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/e420811a04a5437bb723e2085d473cb4 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/e420811a04a5437bb723e2085d473cb4 2024-11-09T02:23:38,366 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 509cbfd6eaffef996b82cfb91370c059/B of 509cbfd6eaffef996b82cfb91370c059 into e420811a04a5437bb723e2085d473cb4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:38,366 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:38,366 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., storeName=509cbfd6eaffef996b82cfb91370c059/B, priority=13, startTime=1731119017938; duration=0sec 2024-11-09T02:23:38,366 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:38,366 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 509cbfd6eaffef996b82cfb91370c059:B 2024-11-09T02:23:38,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T02:23:38,379 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:23:38,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:23:38,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-09T02:23:38,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T02:23:38,381 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:23:38,382 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:23:38,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:23:38,386 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/325ed754411d450cbe1e7ae29e807520 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/325ed754411d450cbe1e7ae29e807520 2024-11-09T02:23:38,389 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 509cbfd6eaffef996b82cfb91370c059/C of 509cbfd6eaffef996b82cfb91370c059 into 325ed754411d450cbe1e7ae29e807520(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:38,389 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:38,389 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., storeName=509cbfd6eaffef996b82cfb91370c059/C, priority=13, startTime=1731119017938; duration=0sec 2024-11-09T02:23:38,389 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:38,389 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 509cbfd6eaffef996b82cfb91370c059:C 2024-11-09T02:23:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T02:23:38,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T02:23:38,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:38,534 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 509cbfd6eaffef996b82cfb91370c059 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-09T02:23:38,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/f36629dd26704d13952a95a5b06f2826 is 50, key is test_row_0/A:col10/1731119016767/Put/seqid=0 2024-11-09T02:23:38,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742299_1475 (size=12151) 2024-11-09T02:23:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T02:23:38,946 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/f36629dd26704d13952a95a5b06f2826 2024-11-09T02:23:38,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/1928aa03c71c421ca695b408905f3546 is 50, key is test_row_0/B:col10/1731119016767/Put/seqid=0 2024-11-09T02:23:38,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:38,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:38,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742300_1476 (size=12151) 2024-11-09T02:23:38,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:38,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119078969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:38,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:38,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:38,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:38,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:38,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119078973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:39,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:39,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46462 deadline: 1731119079000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:39,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:39,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46454 deadline: 1731119079000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:39,004 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_0' in 509cbfd6eaffef996b82cfb91370c059 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:23:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:39,005 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_2' in 509cbfd6eaffef996b82cfb91370c059 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:23:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:39,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T02:23:39,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:39,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46510 deadline: 1731119079020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:39,023 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_0' in 509cbfd6eaffef996b82cfb91370c059 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:23:39,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:39,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:39,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:39,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:39,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119079079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:39,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:39,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:39,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119079079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:39,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:39,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:39,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:39,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:39,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:39,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:39,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:39,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119079290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:39,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119079290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:39,356 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/1928aa03c71c421ca695b408905f3546 2024-11-09T02:23:39,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/0fe3c61ed24a4d3fa2c0a72da49fddc2 is 50, key is test_row_0/C:col10/1731119016767/Put/seqid=0 2024-11-09T02:23:39,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742301_1477 (size=12151) 2024-11-09T02:23:39,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T02:23:39,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:39,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119079599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:39,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:39,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:39,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:39,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119079599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:39,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:39,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:39,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:39,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:39,765 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/0fe3c61ed24a4d3fa2c0a72da49fddc2 2024-11-09T02:23:39,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/f36629dd26704d13952a95a5b06f2826 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/f36629dd26704d13952a95a5b06f2826 2024-11-09T02:23:39,771 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/f36629dd26704d13952a95a5b06f2826, entries=150, sequenceid=244, filesize=11.9 K 2024-11-09T02:23:39,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/1928aa03c71c421ca695b408905f3546 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/1928aa03c71c421ca695b408905f3546 2024-11-09T02:23:39,774 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/1928aa03c71c421ca695b408905f3546, entries=150, sequenceid=244, filesize=11.9 K 2024-11-09T02:23:39,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/0fe3c61ed24a4d3fa2c0a72da49fddc2 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/0fe3c61ed24a4d3fa2c0a72da49fddc2 2024-11-09T02:23:39,777 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/0fe3c61ed24a4d3fa2c0a72da49fddc2, entries=150, sequenceid=244, filesize=11.9 K 2024-11-09T02:23:39,778 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 509cbfd6eaffef996b82cfb91370c059 in 1244ms, sequenceid=244, compaction requested=false 2024-11-09T02:23:39,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:39,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:39,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-09T02:23:39,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-09T02:23:39,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-09T02:23:39,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3970 sec 2024-11-09T02:23:39,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.4010 sec 2024-11-09T02:23:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:40,113 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:23:40,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 509cbfd6eaffef996b82cfb91370c059 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-09T02:23:40,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/5dc69a8266a846119db0c770908b2bb1 is 50, key is test_row_0/A:col10/1731119020112/Put/seqid=0 2024-11-09T02:23:40,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742302_1478 (size=14591) 2024-11-09T02:23:40,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:40,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119080219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:40,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:40,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:40,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:40,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:40,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119080228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:40,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:40,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:40,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:40,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:40,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119080330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:40,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:40,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:40,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:40,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119080340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:40,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:40,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:40,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:40,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/5dc69a8266a846119db0c770908b2bb1 2024-11-09T02:23:40,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/19b60f2f0ae743e08555f91676c8484e is 50, key is test_row_0/B:col10/1731119020112/Put/seqid=0 2024-11-09T02:23:40,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742303_1479 (size=12201) 2024-11-09T02:23:40,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T02:23:40,530 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:23:40,530 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:23:40,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-09T02:23:40,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T02:23:40,532 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:23:40,532 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:23:40,532 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:23:40,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:40,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119080540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:40,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:40,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:40,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:40,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:40,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119080550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:40,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T02:23:40,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-09T02:23:40,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:40,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:40,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:40,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:40,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:40,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:40,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-09T02:23:40,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:40,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:40,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:40,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:40,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:40,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:40,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T02:23:40,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119080849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:40,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:40,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:40,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:40,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:40,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119080860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:40,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:40,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:40,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:40,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/19b60f2f0ae743e08555f91676c8484e 2024-11-09T02:23:40,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/5d1a1f2ef642437fa9419ad8facb32cc is 50, key is test_row_0/C:col10/1731119020112/Put/seqid=0 2024-11-09T02:23:40,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742304_1480 (size=12201) 2024-11-09T02:23:40,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/5d1a1f2ef642437fa9419ad8facb32cc 2024-11-09T02:23:40,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/5dc69a8266a846119db0c770908b2bb1 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/5dc69a8266a846119db0c770908b2bb1 2024-11-09T02:23:40,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/5dc69a8266a846119db0c770908b2bb1, entries=200, sequenceid=257, filesize=14.2 K 2024-11-09T02:23:40,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/19b60f2f0ae743e08555f91676c8484e as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/19b60f2f0ae743e08555f91676c8484e 2024-11-09T02:23:40,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/19b60f2f0ae743e08555f91676c8484e, entries=150, sequenceid=257, filesize=11.9 K 2024-11-09T02:23:40,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/5d1a1f2ef642437fa9419ad8facb32cc as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/5d1a1f2ef642437fa9419ad8facb32cc 2024-11-09T02:23:40,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/5d1a1f2ef642437fa9419ad8facb32cc, entries=150, sequenceid=257, filesize=11.9 K 2024-11-09T02:23:40,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 509cbfd6eaffef996b82cfb91370c059 in 836ms, sequenceid=257, compaction requested=true 2024-11-09T02:23:40,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:40,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 509cbfd6eaffef996b82cfb91370c059:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:23:40,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:40,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 509cbfd6eaffef996b82cfb91370c059:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:23:40,950 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:40,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:40,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 509cbfd6eaffef996b82cfb91370c059:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:23:40,950 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:40,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:40,950 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39439 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:40,950 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:40,950 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): 509cbfd6eaffef996b82cfb91370c059/A is initiating minor compaction (all files) 2024-11-09T02:23:40,950 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 509cbfd6eaffef996b82cfb91370c059/B is initiating minor compaction (all files) 2024-11-09T02:23:40,950 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 509cbfd6eaffef996b82cfb91370c059/B in TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:40,950 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 509cbfd6eaffef996b82cfb91370c059/A in TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:40,951 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/55ae7943f9e34f3698732f2afccfd540, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/f36629dd26704d13952a95a5b06f2826, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/5dc69a8266a846119db0c770908b2bb1] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp, totalSize=38.5 K 2024-11-09T02:23:40,951 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/e420811a04a5437bb723e2085d473cb4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/1928aa03c71c421ca695b408905f3546, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/19b60f2f0ae743e08555f91676c8484e] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp, totalSize=36.2 K 2024-11-09T02:23:40,951 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting e420811a04a5437bb723e2085d473cb4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731119016362 2024-11-09T02:23:40,951 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55ae7943f9e34f3698732f2afccfd540, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731119016362 2024-11-09T02:23:40,951 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 1928aa03c71c421ca695b408905f3546, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731119016765 2024-11-09T02:23:40,951 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting f36629dd26704d13952a95a5b06f2826, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731119016765 2024-11-09T02:23:40,951 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 19b60f2f0ae743e08555f91676c8484e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731119018963 2024-11-09T02:23:40,951 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dc69a8266a846119db0c770908b2bb1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731119018963 2024-11-09T02:23:40,958 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 509cbfd6eaffef996b82cfb91370c059#A#compaction#406 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:40,958 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 509cbfd6eaffef996b82cfb91370c059#B#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:40,958 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/5aaf0be69dda49f6bafdd75e6b0a121d is 50, key is test_row_0/A:col10/1731119020112/Put/seqid=0 2024-11-09T02:23:40,959 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/2677ad9cf2164cb88cd1ab5de5059761 is 50, key is test_row_0/B:col10/1731119020112/Put/seqid=0 2024-11-09T02:23:40,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742306_1482 (size=12849) 2024-11-09T02:23:40,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742305_1481 (size=12849) 2024-11-09T02:23:40,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-09T02:23:40,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:40,990 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 509cbfd6eaffef996b82cfb91370c059 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-09T02:23:40,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/3070e4833c474a86950ccbea5dcc6d4c is 50, key is test_row_0/A:col10/1731119020202/Put/seqid=0 2024-11-09T02:23:40,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742307_1483 (size=12301) 2024-11-09T02:23:41,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T02:23:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:41,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:41,372 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/5aaf0be69dda49f6bafdd75e6b0a121d as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/5aaf0be69dda49f6bafdd75e6b0a121d 2024-11-09T02:23:41,372 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/2677ad9cf2164cb88cd1ab5de5059761 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/2677ad9cf2164cb88cd1ab5de5059761 2024-11-09T02:23:41,376 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 509cbfd6eaffef996b82cfb91370c059/A of 509cbfd6eaffef996b82cfb91370c059 into 5aaf0be69dda49f6bafdd75e6b0a121d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:41,376 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:41,376 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., storeName=509cbfd6eaffef996b82cfb91370c059/A, priority=13, startTime=1731119020950; duration=0sec 2024-11-09T02:23:41,376 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:41,376 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 509cbfd6eaffef996b82cfb91370c059:A 2024-11-09T02:23:41,376 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:41,377 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:41,377 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): 509cbfd6eaffef996b82cfb91370c059/C is initiating minor compaction (all files) 2024-11-09T02:23:41,377 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 509cbfd6eaffef996b82cfb91370c059/C in TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:41,377 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/325ed754411d450cbe1e7ae29e807520, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/0fe3c61ed24a4d3fa2c0a72da49fddc2, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/5d1a1f2ef642437fa9419ad8facb32cc] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp, totalSize=36.2 K 2024-11-09T02:23:41,377 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 325ed754411d450cbe1e7ae29e807520, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731119016362 2024-11-09T02:23:41,378 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fe3c61ed24a4d3fa2c0a72da49fddc2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731119016765 2024-11-09T02:23:41,378 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d1a1f2ef642437fa9419ad8facb32cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731119018963 2024-11-09T02:23:41,379 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 509cbfd6eaffef996b82cfb91370c059/B of 509cbfd6eaffef996b82cfb91370c059 into 2677ad9cf2164cb88cd1ab5de5059761(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:41,379 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:41,379 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., storeName=509cbfd6eaffef996b82cfb91370c059/B, priority=13, startTime=1731119020950; duration=0sec 2024-11-09T02:23:41,379 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:41,379 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 509cbfd6eaffef996b82cfb91370c059:B 2024-11-09T02:23:41,385 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 509cbfd6eaffef996b82cfb91370c059#C#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:41,385 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/824f05ae8eee4c06a90e48419fba6946 is 50, key is test_row_0/C:col10/1731119020112/Put/seqid=0 2024-11-09T02:23:41,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742308_1484 (size=12849) 2024-11-09T02:23:41,398 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/3070e4833c474a86950ccbea5dcc6d4c 2024-11-09T02:23:41,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119081396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:41,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:41,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:41,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:41,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/92b79203fe7a471c96ab7427e0e8302c is 50, key is test_row_0/B:col10/1731119020202/Put/seqid=0 2024-11-09T02:23:41,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119081397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:41,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742309_1485 (size=12301) 2024-11-09T02:23:41,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:41,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119081510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:41,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:41,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119081510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:41,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:41,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:41,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:41,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:41,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:41,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T02:23:41,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:41,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119081720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:41,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:41,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:41,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119081720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:41,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:41,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:41,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:41,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:41,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:41,793 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/824f05ae8eee4c06a90e48419fba6946 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/824f05ae8eee4c06a90e48419fba6946 2024-11-09T02:23:41,796 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 509cbfd6eaffef996b82cfb91370c059/C of 509cbfd6eaffef996b82cfb91370c059 into 824f05ae8eee4c06a90e48419fba6946(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:41,796 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:41,796 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., storeName=509cbfd6eaffef996b82cfb91370c059/C, priority=13, startTime=1731119020950; duration=0sec 2024-11-09T02:23:41,796 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:41,796 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 509cbfd6eaffef996b82cfb91370c059:C 2024-11-09T02:23:41,809 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/92b79203fe7a471c96ab7427e0e8302c 2024-11-09T02:23:41,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/70a3da9b5d994bc1a091b38cc84d99de is 50, key is test_row_0/C:col10/1731119020202/Put/seqid=0 2024-11-09T02:23:41,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742310_1486 (size=12301) 2024-11-09T02:23:42,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:42,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119082030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:42,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:42,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119082030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:42,219 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/70a3da9b5d994bc1a091b38cc84d99de 2024-11-09T02:23:42,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/3070e4833c474a86950ccbea5dcc6d4c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/3070e4833c474a86950ccbea5dcc6d4c 2024-11-09T02:23:42,225 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/3070e4833c474a86950ccbea5dcc6d4c, entries=150, sequenceid=280, filesize=12.0 K 2024-11-09T02:23:42,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/92b79203fe7a471c96ab7427e0e8302c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/92b79203fe7a471c96ab7427e0e8302c 2024-11-09T02:23:42,228 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/92b79203fe7a471c96ab7427e0e8302c, entries=150, sequenceid=280, filesize=12.0 K 2024-11-09T02:23:42,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/70a3da9b5d994bc1a091b38cc84d99de as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/70a3da9b5d994bc1a091b38cc84d99de 2024-11-09T02:23:42,231 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/70a3da9b5d994bc1a091b38cc84d99de, entries=150, sequenceid=280, filesize=12.0 K 2024-11-09T02:23:42,231 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 509cbfd6eaffef996b82cfb91370c059 in 1242ms, sequenceid=280, compaction requested=false 2024-11-09T02:23:42,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:42,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:42,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-09T02:23:42,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-09T02:23:42,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-09T02:23:42,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7000 sec 2024-11-09T02:23:42,235 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.7040 sec 2024-11-09T02:23:42,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:42,544 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:23:42,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 509cbfd6eaffef996b82cfb91370c059 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-09T02:23:42,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/d97e189e5e4041968fc098eda6554baa is 50, key is test_row_0/A:col10/1731119021395/Put/seqid=0 2024-11-09T02:23:42,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742311_1487 (size=14741) 2024-11-09T02:23:42,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119082609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:42,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:42,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:42,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:42,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119082610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:42,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T02:23:42,680 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:23:42,681 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:23:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-09T02:23:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T02:23:42,682 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:23:42,683 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:23:42,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:23:42,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119082719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:42,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:42,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:42,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:42,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119082730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:42,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:42,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:42,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:42,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T02:23:42,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T02:23:42,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:42,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:42,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:42,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:42,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:42,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:42,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:42,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119082929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:42,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:42,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:42,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:42,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:42,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119082940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:42,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:42,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:42,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:42,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/d97e189e5e4041968fc098eda6554baa 2024-11-09T02:23:42,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/6783ba438110487ea65a053afce9a7a8 is 50, key is test_row_0/B:col10/1731119021395/Put/seqid=0 2024-11-09T02:23:42,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742312_1488 (size=12301) 2024-11-09T02:23:42,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T02:23:42,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:42,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:42,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:42,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:42,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:43,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T02:23:43,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T02:23:43,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:43,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:43,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:43,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:43,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:43,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:43,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:43,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119083240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:43,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:43,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:43,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:43,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:43,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119083260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:43,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:43,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:43,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:43,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T02:23:43,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:43,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:43,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:43,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:43,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:43,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:43,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T02:23:43,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/6783ba438110487ea65a053afce9a7a8 2024-11-09T02:23:43,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/200c80185aae44d69607dbb7470b9eb0 is 50, key is test_row_0/C:col10/1731119021395/Put/seqid=0 2024-11-09T02:23:43,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742313_1489 (size=12301) 2024-11-09T02:23:43,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T02:23:43,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:43,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:43,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:43,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:43,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:43,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:43,499 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T02:23:43,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T02:23:43,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:43,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:43,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:43,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:43,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:43,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:43,609 INFO [Thread-1926 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1926. 2024-11-09T02:23:43,609 DEBUG [Thread-1926 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:23:43,609 DEBUG [Thread-1926 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:43,609 DEBUG [Thread-1926 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:43,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:23:43,610 INFO [Thread-1930 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1930. 2024-11-09T02:23:43,610 DEBUG [Thread-1930 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:23:43,610 DEBUG [Thread-1930 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:43,610 DEBUG [Thread-1930 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:43,610 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:23:43,612 INFO [Thread-1928 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1928. 2024-11-09T02:23:43,612 DEBUG [Thread-1928 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:23:43,612 DEBUG [Thread-1928 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:43,612 DEBUG [Thread-1928 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:43,612 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:23:43,618 INFO [Thread-1932 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1932. 2024-11-09T02:23:43,618 DEBUG [Thread-1932 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:23:43,618 DEBUG [Thread-1932 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:43,618 DEBUG [Thread-1932 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:43,618 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:23:43,618 INFO [Thread-1924 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1924. 2024-11-09T02:23:43,618 DEBUG [Thread-1924 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:23:43,618 DEBUG [Thread-1924 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:43,618 DEBUG [Thread-1924 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:43,618 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:23:43,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T02:23:43,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:43,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:43,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46446 deadline: 1731119083751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:43,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:43,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:43,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:43,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/200c80185aae44d69607dbb7470b9eb0 2024-11-09T02:23:43,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/d97e189e5e4041968fc098eda6554baa as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/d97e189e5e4041968fc098eda6554baa 2024-11-09T02:23:43,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46520 deadline: 1731119083780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:43,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:43,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:43,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., hostname=53f286d3af3b,46079,1731118875223, seqNum=2 because the exception is null or not the one we care about 2024-11-09T02:23:43,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/d97e189e5e4041968fc098eda6554baa, entries=200, sequenceid=297, filesize=14.4 K 2024-11-09T02:23:43,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/6783ba438110487ea65a053afce9a7a8 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/6783ba438110487ea65a053afce9a7a8 2024-11-09T02:23:43,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/6783ba438110487ea65a053afce9a7a8, entries=150, sequenceid=297, filesize=12.0 K 2024-11-09T02:23:43,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/200c80185aae44d69607dbb7470b9eb0 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/200c80185aae44d69607dbb7470b9eb0 2024-11-09T02:23:43,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/200c80185aae44d69607dbb7470b9eb0, entries=150, sequenceid=297, filesize=12.0 K 2024-11-09T02:23:43,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 509cbfd6eaffef996b82cfb91370c059 in 1250ms, sequenceid=297, compaction requested=true 2024-11-09T02:23:43,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:43,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 509cbfd6eaffef996b82cfb91370c059:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:23:43,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:43,794 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:43,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 509cbfd6eaffef996b82cfb91370c059:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:23:43,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:43,794 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:43,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 509cbfd6eaffef996b82cfb91370c059:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:23:43,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:43,795 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:43,795 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:43,795 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): 509cbfd6eaffef996b82cfb91370c059/B is initiating minor compaction (all files) 2024-11-09T02:23:43,795 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 509cbfd6eaffef996b82cfb91370c059/A is initiating minor compaction (all files) 2024-11-09T02:23:43,795 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 509cbfd6eaffef996b82cfb91370c059/A in TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:43,795 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 509cbfd6eaffef996b82cfb91370c059/B in TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:43,795 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/5aaf0be69dda49f6bafdd75e6b0a121d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/3070e4833c474a86950ccbea5dcc6d4c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/d97e189e5e4041968fc098eda6554baa] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp, totalSize=39.0 K 2024-11-09T02:23:43,795 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/2677ad9cf2164cb88cd1ab5de5059761, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/92b79203fe7a471c96ab7427e0e8302c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/6783ba438110487ea65a053afce9a7a8] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp, totalSize=36.6 K 2024-11-09T02:23:43,795 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2677ad9cf2164cb88cd1ab5de5059761, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731119018963 2024-11-09T02:23:43,795 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 5aaf0be69dda49f6bafdd75e6b0a121d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731119018963 2024-11-09T02:23:43,795 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 3070e4833c474a86950ccbea5dcc6d4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731119020193 2024-11-09T02:23:43,795 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92b79203fe7a471c96ab7427e0e8302c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731119020193 2024-11-09T02:23:43,796 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting d97e189e5e4041968fc098eda6554baa, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731119021377 2024-11-09T02:23:43,796 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6783ba438110487ea65a053afce9a7a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731119021389 2024-11-09T02:23:43,801 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 509cbfd6eaffef996b82cfb91370c059#B#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:43,801 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 509cbfd6eaffef996b82cfb91370c059#A#compaction#415 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:43,802 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/fcccbde373a442d380a8bb7801837b7c is 50, key is test_row_0/B:col10/1731119021395/Put/seqid=0 2024-11-09T02:23:43,802 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/69b9e4e5d3a6407598fe75ca83df9948 is 50, key is test_row_0/A:col10/1731119021395/Put/seqid=0 2024-11-09T02:23:43,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742315_1491 (size=13051) 2024-11-09T02:23:43,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742314_1490 (size=13051) 2024-11-09T02:23:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T02:23:43,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T02:23:43,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:43,908 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 509cbfd6eaffef996b82cfb91370c059 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-09T02:23:43,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/651be8d91a0d40d988b5c52d9ded93ed is 50, key is test_row_0/A:col10/1731119022608/Put/seqid=0 2024-11-09T02:23:43,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742316_1492 (size=12301) 2024-11-09T02:23:44,266 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/fcccbde373a442d380a8bb7801837b7c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/fcccbde373a442d380a8bb7801837b7c 2024-11-09T02:23:44,266 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/69b9e4e5d3a6407598fe75ca83df9948 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/69b9e4e5d3a6407598fe75ca83df9948 2024-11-09T02:23:44,275 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 509cbfd6eaffef996b82cfb91370c059/A of 509cbfd6eaffef996b82cfb91370c059 into 69b9e4e5d3a6407598fe75ca83df9948(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:44,275 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 509cbfd6eaffef996b82cfb91370c059/B of 509cbfd6eaffef996b82cfb91370c059 into fcccbde373a442d380a8bb7801837b7c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:44,276 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:44,276 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:44,276 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., storeName=509cbfd6eaffef996b82cfb91370c059/B, priority=13, startTime=1731119023794; duration=0sec 2024-11-09T02:23:44,276 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., storeName=509cbfd6eaffef996b82cfb91370c059/A, priority=13, startTime=1731119023794; duration=0sec 2024-11-09T02:23:44,276 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:44,276 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 509cbfd6eaffef996b82cfb91370c059:A 2024-11-09T02:23:44,276 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:44,276 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 509cbfd6eaffef996b82cfb91370c059:B 2024-11-09T02:23:44,276 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:44,277 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:44,277 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 509cbfd6eaffef996b82cfb91370c059/C is initiating minor compaction (all files) 2024-11-09T02:23:44,277 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 509cbfd6eaffef996b82cfb91370c059/C in TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:44,277 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/824f05ae8eee4c06a90e48419fba6946, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/70a3da9b5d994bc1a091b38cc84d99de, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/200c80185aae44d69607dbb7470b9eb0] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp, totalSize=36.6 K 2024-11-09T02:23:44,278 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 824f05ae8eee4c06a90e48419fba6946, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731119018963 2024-11-09T02:23:44,278 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 70a3da9b5d994bc1a091b38cc84d99de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731119020193 2024-11-09T02:23:44,278 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 200c80185aae44d69607dbb7470b9eb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731119021389 2024-11-09T02:23:44,283 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 509cbfd6eaffef996b82cfb91370c059#C#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:44,284 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/c051cdd5edb44d65b5f92be0ff96b9d5 is 50, key is test_row_0/C:col10/1731119021395/Put/seqid=0 2024-11-09T02:23:44,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742317_1493 (size=13051) 2024-11-09T02:23:44,320 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/651be8d91a0d40d988b5c52d9ded93ed 2024-11-09T02:23:44,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/e4586501584d48da8dd0b866cd6c1c34 is 50, key is test_row_0/B:col10/1731119022608/Put/seqid=0 2024-11-09T02:23:44,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742318_1494 (size=12301) 2024-11-09T02:23:44,699 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/c051cdd5edb44d65b5f92be0ff96b9d5 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/c051cdd5edb44d65b5f92be0ff96b9d5 2024-11-09T02:23:44,705 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 509cbfd6eaffef996b82cfb91370c059/C of 509cbfd6eaffef996b82cfb91370c059 into c051cdd5edb44d65b5f92be0ff96b9d5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:44,705 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:44,705 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., storeName=509cbfd6eaffef996b82cfb91370c059/C, priority=13, startTime=1731119023794; duration=0sec 2024-11-09T02:23:44,705 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:44,706 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 509cbfd6eaffef996b82cfb91370c059:C 2024-11-09T02:23:44,728 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/e4586501584d48da8dd0b866cd6c1c34 2024-11-09T02:23:44,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/0f8f2203b5f74f808a68c6b923d81fc1 is 50, key is test_row_0/C:col10/1731119022608/Put/seqid=0 2024-11-09T02:23:44,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742319_1495 (size=12301) 2024-11-09T02:23:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:44,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. as already flushing 2024-11-09T02:23:44,762 INFO [Thread-1913 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1913. 2024-11-09T02:23:44,762 DEBUG [Thread-1913 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:23:44,762 DEBUG [Thread-1913 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:44,763 DEBUG [Thread-1913 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:44,763 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:23:44,793 INFO [Thread-1919 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1919. 2024-11-09T02:23:44,794 DEBUG [Thread-1919 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:23:44,794 DEBUG [Thread-1919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:44,794 DEBUG [Thread-1919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:44,794 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:23:44,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T02:23:45,139 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/0f8f2203b5f74f808a68c6b923d81fc1 2024-11-09T02:23:45,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/651be8d91a0d40d988b5c52d9ded93ed as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/651be8d91a0d40d988b5c52d9ded93ed 2024-11-09T02:23:45,152 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/651be8d91a0d40d988b5c52d9ded93ed, entries=150, sequenceid=319, filesize=12.0 K 2024-11-09T02:23:45,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/e4586501584d48da8dd0b866cd6c1c34 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/e4586501584d48da8dd0b866cd6c1c34 2024-11-09T02:23:45,157 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/e4586501584d48da8dd0b866cd6c1c34, entries=150, sequenceid=319, filesize=12.0 K 2024-11-09T02:23:45,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/0f8f2203b5f74f808a68c6b923d81fc1 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/0f8f2203b5f74f808a68c6b923d81fc1 2024-11-09T02:23:45,161 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/0f8f2203b5f74f808a68c6b923d81fc1, entries=150, sequenceid=319, filesize=12.0 K 2024-11-09T02:23:45,161 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=13.42 KB/13740 for 509cbfd6eaffef996b82cfb91370c059 in 1254ms, sequenceid=319, compaction requested=false 2024-11-09T02:23:45,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 509cbfd6eaffef996b82cfb91370c059: 2024-11-09T02:23:45,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:45,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-09T02:23:45,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-09T02:23:45,165 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-09T02:23:45,165 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4800 sec 2024-11-09T02:23:45,166 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.4840 sec 2024-11-09T02:23:46,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T02:23:46,852 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:23:49,064 INFO [Thread-1917 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1917. 2024-11-09T02:23:49,064 DEBUG [Thread-1917 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:23:49,064 DEBUG [Thread-1917 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:49,065 DEBUG [Thread-1917 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:49,065 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:23:49,073 INFO [Thread-1915 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1915. 2024-11-09T02:23:49,073 DEBUG [Thread-1915 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:23:49,073 DEBUG [Thread-1915 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:49,073 DEBUG [Thread-1915 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:49,073 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:23:49,122 INFO [Thread-1921 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1921. 2024-11-09T02:23:49,122 DEBUG [Thread-1921 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:23:49,122 DEBUG [Thread-1921 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:49,122 DEBUG [Thread-1921 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:49,123 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:23:49,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T02:23:49,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-09T02:23:49,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-09T02:23:49,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-09T02:23:49,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-09T02:23:49,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-09T02:23:49,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T02:23:49,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T02:23:49,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1774 2024-11-09T02:23:49,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5322 rows 2024-11-09T02:23:49,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1780 2024-11-09T02:23:49,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5340 rows 2024-11-09T02:23:49,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1777 2024-11-09T02:23:49,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5331 rows 2024-11-09T02:23:49,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1780 2024-11-09T02:23:49,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5338 rows 2024-11-09T02:23:49,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1772 2024-11-09T02:23:49,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5316 rows 2024-11-09T02:23:49,123 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T02:23:49,124 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:23:49,124 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:49,124 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:49,124 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:23:49,125 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T02:23:49,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T02:23:49,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T02:23:49,129 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731119029128"}]},"ts":"1731119029128"} 2024-11-09T02:23:49,130 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T02:23:49,130 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T02:23:49,131 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T02:23:49,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=509cbfd6eaffef996b82cfb91370c059, UNASSIGN}] 2024-11-09T02:23:49,134 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=509cbfd6eaffef996b82cfb91370c059, UNASSIGN 2024-11-09T02:23:49,135 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=509cbfd6eaffef996b82cfb91370c059, regionState=CLOSING, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:49,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=509cbfd6eaffef996b82cfb91370c059, UNASSIGN because future has completed 2024-11-09T02:23:49,136 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T02:23:49,136 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure 509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223}] 2024-11-09T02:23:49,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T02:23:49,290 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(122): Close 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:49,290 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T02:23:49,290 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1722): Closing 509cbfd6eaffef996b82cfb91370c059, disabling compactions & flushes 2024-11-09T02:23:49,290 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:49,290 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:49,290 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. after waiting 0 ms 2024-11-09T02:23:49,290 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:49,291 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(2902): Flushing 509cbfd6eaffef996b82cfb91370c059 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-09T02:23:49,299 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/7c0b186b8a674e0eb40659277eb6f9b5 is 50, key is test_row_0/A:col10/1731119029121/Put/seqid=0 2024-11-09T02:23:49,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742320_1496 (size=9857) 2024-11-09T02:23:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T02:23:49,705 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/7c0b186b8a674e0eb40659277eb6f9b5 2024-11-09T02:23:49,713 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/8a778dd0367442e88e65aea1dc74ca55 is 50, key is test_row_0/B:col10/1731119029121/Put/seqid=0 2024-11-09T02:23:49,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742321_1497 (size=9857) 2024-11-09T02:23:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T02:23:50,119 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/8a778dd0367442e88e65aea1dc74ca55 2024-11-09T02:23:50,133 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/60313ac4054f4f1e8e9d12937870f19a is 50, key is test_row_0/C:col10/1731119029121/Put/seqid=0 2024-11-09T02:23:50,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742322_1498 (size=9857) 2024-11-09T02:23:50,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T02:23:50,538 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/60313ac4054f4f1e8e9d12937870f19a 2024-11-09T02:23:50,549 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/A/7c0b186b8a674e0eb40659277eb6f9b5 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/7c0b186b8a674e0eb40659277eb6f9b5 2024-11-09T02:23:50,555 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/7c0b186b8a674e0eb40659277eb6f9b5, entries=100, sequenceid=330, filesize=9.6 K 2024-11-09T02:23:50,556 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/B/8a778dd0367442e88e65aea1dc74ca55 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/8a778dd0367442e88e65aea1dc74ca55 2024-11-09T02:23:50,561 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/8a778dd0367442e88e65aea1dc74ca55, entries=100, sequenceid=330, filesize=9.6 K 2024-11-09T02:23:50,562 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/.tmp/C/60313ac4054f4f1e8e9d12937870f19a as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/60313ac4054f4f1e8e9d12937870f19a 2024-11-09T02:23:50,566 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/60313ac4054f4f1e8e9d12937870f19a, entries=100, sequenceid=330, filesize=9.6 K 2024-11-09T02:23:50,566 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 509cbfd6eaffef996b82cfb91370c059 in 1276ms, sequenceid=330, compaction requested=true 2024-11-09T02:23:50,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/bbf44f554b7a43bb824110756204e716, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/d9ec5f9f05f14dfea463effc6c181b3c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/53225891130646c5aeeea9b801072fff, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/a689efc4e503418283f0cd61911fca42, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/81dd713d3af44553a5e0ff1f8cecde75, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/dff49072ed464aeaa0ce3198abc6f88d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/1e42d20e84254a4eac7dd761390ad273, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/96427184ca8345bbaec4629b1280ff70, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/dd1c6355e4994a5ca6748af354d5f61e, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/5b2c20e368814c6287232d24e9b97545, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/b4592cc9cba44276a711bd9dc2ca85ba, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/8b8654ddd61042ee96e2073f0e7c8e4c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/20d0904feb8e4c66a3ead86fefaeab72, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/7bddb510e61a48fabd73c435dc8a03f4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/2534285fdd74423d820f12f4607c22ba, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/b4fa37fbb1ea46b084552e391772fcaa, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/55ae7943f9e34f3698732f2afccfd540, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/f36629dd26704d13952a95a5b06f2826, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/5dc69a8266a846119db0c770908b2bb1, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/5aaf0be69dda49f6bafdd75e6b0a121d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/3070e4833c474a86950ccbea5dcc6d4c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/d97e189e5e4041968fc098eda6554baa] to archive 2024-11-09T02:23:50,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T02:23:50,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/bbf44f554b7a43bb824110756204e716 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/bbf44f554b7a43bb824110756204e716 2024-11-09T02:23:50,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/d9ec5f9f05f14dfea463effc6c181b3c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/d9ec5f9f05f14dfea463effc6c181b3c 2024-11-09T02:23:50,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/53225891130646c5aeeea9b801072fff to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/53225891130646c5aeeea9b801072fff 2024-11-09T02:23:50,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/a689efc4e503418283f0cd61911fca42 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/a689efc4e503418283f0cd61911fca42 2024-11-09T02:23:50,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/81dd713d3af44553a5e0ff1f8cecde75 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/81dd713d3af44553a5e0ff1f8cecde75 2024-11-09T02:23:50,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/dff49072ed464aeaa0ce3198abc6f88d to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/dff49072ed464aeaa0ce3198abc6f88d 2024-11-09T02:23:50,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/1e42d20e84254a4eac7dd761390ad273 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/1e42d20e84254a4eac7dd761390ad273 2024-11-09T02:23:50,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/96427184ca8345bbaec4629b1280ff70 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/96427184ca8345bbaec4629b1280ff70 2024-11-09T02:23:50,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/dd1c6355e4994a5ca6748af354d5f61e to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/dd1c6355e4994a5ca6748af354d5f61e 2024-11-09T02:23:50,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/5b2c20e368814c6287232d24e9b97545 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/5b2c20e368814c6287232d24e9b97545 2024-11-09T02:23:50,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/b4592cc9cba44276a711bd9dc2ca85ba to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/b4592cc9cba44276a711bd9dc2ca85ba 2024-11-09T02:23:50,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/8b8654ddd61042ee96e2073f0e7c8e4c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/8b8654ddd61042ee96e2073f0e7c8e4c 2024-11-09T02:23:50,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/20d0904feb8e4c66a3ead86fefaeab72 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/20d0904feb8e4c66a3ead86fefaeab72 2024-11-09T02:23:50,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/7bddb510e61a48fabd73c435dc8a03f4 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/7bddb510e61a48fabd73c435dc8a03f4 2024-11-09T02:23:50,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/2534285fdd74423d820f12f4607c22ba to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/2534285fdd74423d820f12f4607c22ba 2024-11-09T02:23:50,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/b4fa37fbb1ea46b084552e391772fcaa to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/b4fa37fbb1ea46b084552e391772fcaa 2024-11-09T02:23:50,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/55ae7943f9e34f3698732f2afccfd540 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/55ae7943f9e34f3698732f2afccfd540 2024-11-09T02:23:50,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/f36629dd26704d13952a95a5b06f2826 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/f36629dd26704d13952a95a5b06f2826 2024-11-09T02:23:50,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/5dc69a8266a846119db0c770908b2bb1 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/5dc69a8266a846119db0c770908b2bb1 2024-11-09T02:23:50,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/5aaf0be69dda49f6bafdd75e6b0a121d to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/5aaf0be69dda49f6bafdd75e6b0a121d 2024-11-09T02:23:50,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/3070e4833c474a86950ccbea5dcc6d4c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/3070e4833c474a86950ccbea5dcc6d4c 2024-11-09T02:23:50,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/d97e189e5e4041968fc098eda6554baa to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/d97e189e5e4041968fc098eda6554baa 2024-11-09T02:23:50,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/d46220d0d4c441e19ada3c31985e8258, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/e9d3a51523454f1d8bdcfd56affc0374, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/887679cccf654a308af145725c2eea62, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/1b66f87dbabf48d3809dd91b6e26bf41, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/d56eb7c26e9446efb62e691efb8506f6, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/e172b1f70990429a886e7d52d9035ff6, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/52972580832948f1867a3d327503ed0e, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/3b7c063c9b874bf6bf1489acc59c6670, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/f4c33c0247154cb68db4403bc3a3f8c3, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/5d97933de84d4cabb4fb21a49da8cf17, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/06b32d9fb1ad4209b2b74be29a3c7752, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/d50e4bf8615a40f7800a2dc027ce8e36, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/322ea8d3c0ae46bd9ca4c1b4916855ac, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/6bc9001c99f94f0ab9ec8f264f5138f0, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/d971da954c114578bd6cad02619b27aa, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/e420811a04a5437bb723e2085d473cb4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/5e0698052ede45148a21bc6f32508d2d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/1928aa03c71c421ca695b408905f3546, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/2677ad9cf2164cb88cd1ab5de5059761, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/19b60f2f0ae743e08555f91676c8484e, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/92b79203fe7a471c96ab7427e0e8302c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/6783ba438110487ea65a053afce9a7a8] to archive 2024-11-09T02:23:50,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T02:23:50,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/d46220d0d4c441e19ada3c31985e8258 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/d46220d0d4c441e19ada3c31985e8258 2024-11-09T02:23:50,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/e9d3a51523454f1d8bdcfd56affc0374 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/e9d3a51523454f1d8bdcfd56affc0374 2024-11-09T02:23:50,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/887679cccf654a308af145725c2eea62 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/887679cccf654a308af145725c2eea62 2024-11-09T02:23:50,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/1b66f87dbabf48d3809dd91b6e26bf41 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/1b66f87dbabf48d3809dd91b6e26bf41 2024-11-09T02:23:50,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/d56eb7c26e9446efb62e691efb8506f6 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/d56eb7c26e9446efb62e691efb8506f6 2024-11-09T02:23:50,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/e172b1f70990429a886e7d52d9035ff6 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/e172b1f70990429a886e7d52d9035ff6 2024-11-09T02:23:50,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/52972580832948f1867a3d327503ed0e to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/52972580832948f1867a3d327503ed0e 2024-11-09T02:23:50,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/3b7c063c9b874bf6bf1489acc59c6670 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/3b7c063c9b874bf6bf1489acc59c6670 2024-11-09T02:23:50,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/f4c33c0247154cb68db4403bc3a3f8c3 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/f4c33c0247154cb68db4403bc3a3f8c3 2024-11-09T02:23:50,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/5d97933de84d4cabb4fb21a49da8cf17 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/5d97933de84d4cabb4fb21a49da8cf17 2024-11-09T02:23:50,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/06b32d9fb1ad4209b2b74be29a3c7752 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/06b32d9fb1ad4209b2b74be29a3c7752 2024-11-09T02:23:50,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/d50e4bf8615a40f7800a2dc027ce8e36 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/d50e4bf8615a40f7800a2dc027ce8e36 2024-11-09T02:23:50,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/322ea8d3c0ae46bd9ca4c1b4916855ac to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/322ea8d3c0ae46bd9ca4c1b4916855ac 2024-11-09T02:23:50,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/6bc9001c99f94f0ab9ec8f264f5138f0 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/6bc9001c99f94f0ab9ec8f264f5138f0 2024-11-09T02:23:50,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/d971da954c114578bd6cad02619b27aa to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/d971da954c114578bd6cad02619b27aa 2024-11-09T02:23:50,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/e420811a04a5437bb723e2085d473cb4 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/e420811a04a5437bb723e2085d473cb4 2024-11-09T02:23:50,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/5e0698052ede45148a21bc6f32508d2d to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/5e0698052ede45148a21bc6f32508d2d 2024-11-09T02:23:50,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/1928aa03c71c421ca695b408905f3546 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/1928aa03c71c421ca695b408905f3546 2024-11-09T02:23:50,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/2677ad9cf2164cb88cd1ab5de5059761 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/2677ad9cf2164cb88cd1ab5de5059761 2024-11-09T02:23:50,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/19b60f2f0ae743e08555f91676c8484e to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/19b60f2f0ae743e08555f91676c8484e 2024-11-09T02:23:50,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/92b79203fe7a471c96ab7427e0e8302c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/92b79203fe7a471c96ab7427e0e8302c 2024-11-09T02:23:50,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/6783ba438110487ea65a053afce9a7a8 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/6783ba438110487ea65a053afce9a7a8 2024-11-09T02:23:50,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/993ae7e4ffdd404887a4d22942b2c3ab, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/612507cb595c40c29814c11c46d8d59f, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/ae3ec774ca2b4a0e9b6e48ded1d9c63e, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/f5a11b46c61e435191f21c4ee19ab2e3, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/96e73283ac714103be9d0902b9b900c4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/7bf52aefda72452e9ed0bcfa66ecf06d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/0be877724b254839b78e49136894e077, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/116967be1dc8424b8b42f7b7ac76c76a, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/349ac1735b0c4279aa201b480a2ada71, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/e764533927af408a8271f8e0936fbb44, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/788a35eddbf54d84a0bb9a36819923fa, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/b95b6474144946248b4181023b77ed16, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/92268d3e35494456a51655f8d1e612e7, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/076a511eafb14bf995a2e6bb74a93463, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/3e58c650b0e144b991233dcc2958be2c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/325ed754411d450cbe1e7ae29e807520, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/baceb1e3d3b5490cb8da8da4af75a0b4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/0fe3c61ed24a4d3fa2c0a72da49fddc2, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/824f05ae8eee4c06a90e48419fba6946, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/5d1a1f2ef642437fa9419ad8facb32cc, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/70a3da9b5d994bc1a091b38cc84d99de, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/200c80185aae44d69607dbb7470b9eb0] to archive 2024-11-09T02:23:50,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T02:23:50,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/993ae7e4ffdd404887a4d22942b2c3ab to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/993ae7e4ffdd404887a4d22942b2c3ab 2024-11-09T02:23:50,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/612507cb595c40c29814c11c46d8d59f to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/612507cb595c40c29814c11c46d8d59f 2024-11-09T02:23:50,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/ae3ec774ca2b4a0e9b6e48ded1d9c63e to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/ae3ec774ca2b4a0e9b6e48ded1d9c63e 2024-11-09T02:23:50,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/f5a11b46c61e435191f21c4ee19ab2e3 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/f5a11b46c61e435191f21c4ee19ab2e3 2024-11-09T02:23:50,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/96e73283ac714103be9d0902b9b900c4 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/96e73283ac714103be9d0902b9b900c4 2024-11-09T02:23:50,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/7bf52aefda72452e9ed0bcfa66ecf06d to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/7bf52aefda72452e9ed0bcfa66ecf06d 2024-11-09T02:23:50,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/0be877724b254839b78e49136894e077 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/0be877724b254839b78e49136894e077 2024-11-09T02:23:50,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/116967be1dc8424b8b42f7b7ac76c76a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/116967be1dc8424b8b42f7b7ac76c76a 2024-11-09T02:23:50,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/349ac1735b0c4279aa201b480a2ada71 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/349ac1735b0c4279aa201b480a2ada71 2024-11-09T02:23:50,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/e764533927af408a8271f8e0936fbb44 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/e764533927af408a8271f8e0936fbb44 2024-11-09T02:23:50,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/788a35eddbf54d84a0bb9a36819923fa to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/788a35eddbf54d84a0bb9a36819923fa 2024-11-09T02:23:50,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/b95b6474144946248b4181023b77ed16 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/b95b6474144946248b4181023b77ed16 2024-11-09T02:23:50,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/92268d3e35494456a51655f8d1e612e7 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/92268d3e35494456a51655f8d1e612e7 2024-11-09T02:23:50,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/076a511eafb14bf995a2e6bb74a93463 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/076a511eafb14bf995a2e6bb74a93463 2024-11-09T02:23:50,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/3e58c650b0e144b991233dcc2958be2c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/3e58c650b0e144b991233dcc2958be2c 2024-11-09T02:23:50,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/325ed754411d450cbe1e7ae29e807520 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/325ed754411d450cbe1e7ae29e807520 2024-11-09T02:23:50,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/baceb1e3d3b5490cb8da8da4af75a0b4 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/baceb1e3d3b5490cb8da8da4af75a0b4 2024-11-09T02:23:50,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/0fe3c61ed24a4d3fa2c0a72da49fddc2 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/0fe3c61ed24a4d3fa2c0a72da49fddc2 2024-11-09T02:23:50,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/824f05ae8eee4c06a90e48419fba6946 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/824f05ae8eee4c06a90e48419fba6946 2024-11-09T02:23:50,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/5d1a1f2ef642437fa9419ad8facb32cc to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/5d1a1f2ef642437fa9419ad8facb32cc 2024-11-09T02:23:50,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/70a3da9b5d994bc1a091b38cc84d99de to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/70a3da9b5d994bc1a091b38cc84d99de 2024-11-09T02:23:50,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/200c80185aae44d69607dbb7470b9eb0 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/200c80185aae44d69607dbb7470b9eb0 2024-11-09T02:23:50,638 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/recovered.edits/333.seqid, newMaxSeqId=333, maxSeqId=1 2024-11-09T02:23:50,639 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. 2024-11-09T02:23:50,639 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1676): Region close journal for 509cbfd6eaffef996b82cfb91370c059: Waiting for close lock at 1731119029290Running coprocessor pre-close hooks at 1731119029290Disabling compacts and flushes for region at 1731119029290Disabling writes for close at 1731119029290Obtaining lock to block concurrent updates at 1731119029291 (+1 ms)Preparing flush snapshotting stores in 509cbfd6eaffef996b82cfb91370c059 at 1731119029291Finished memstore snapshotting TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731119029291Flushing stores of TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059. at 1731119029293 (+2 ms)Flushing 509cbfd6eaffef996b82cfb91370c059/A: creating writer at 1731119029293Flushing 509cbfd6eaffef996b82cfb91370c059/A: appending metadata at 1731119029298 (+5 ms)Flushing 509cbfd6eaffef996b82cfb91370c059/A: closing flushed file at 1731119029298Flushing 509cbfd6eaffef996b82cfb91370c059/B: creating writer at 1731119029709 (+411 ms)Flushing 509cbfd6eaffef996b82cfb91370c059/B: appending metadata at 1731119029712 (+3 ms)Flushing 509cbfd6eaffef996b82cfb91370c059/B: closing flushed file at 1731119029712Flushing 509cbfd6eaffef996b82cfb91370c059/C: creating writer at 1731119030127 (+415 ms)Flushing 509cbfd6eaffef996b82cfb91370c059/C: appending metadata at 1731119030132 (+5 ms)Flushing 509cbfd6eaffef996b82cfb91370c059/C: closing flushed file at 1731119030132Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58d42d06: reopening flushed file at 1731119030547 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3831db87: reopening flushed file at 1731119030555 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68918d14: reopening flushed file at 1731119030561 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 509cbfd6eaffef996b82cfb91370c059 in 1276ms, sequenceid=330, compaction requested=true at 1731119030566 (+5 ms)Writing region close event to WAL at 1731119030635 (+69 ms)Running coprocessor post-close hooks at 1731119030639 (+4 ms)Closed at 1731119030639 2024-11-09T02:23:50,641 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(157): Closed 509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:50,641 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=509cbfd6eaffef996b82cfb91370c059, regionState=CLOSED 2024-11-09T02:23:50,642 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure 509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 because future has completed 2024-11-09T02:23:50,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-09T02:23:50,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseRegionProcedure 509cbfd6eaffef996b82cfb91370c059, server=53f286d3af3b,46079,1731118875223 in 1.5070 sec 2024-11-09T02:23:50,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-09T02:23:50,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=509cbfd6eaffef996b82cfb91370c059, UNASSIGN in 1.5110 sec 2024-11-09T02:23:50,647 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-09T02:23:50,647 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5150 sec 2024-11-09T02:23:50,648 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731119030648"}]},"ts":"1731119030648"} 2024-11-09T02:23:50,649 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T02:23:50,649 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T02:23:50,650 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5240 sec 2024-11-09T02:23:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T02:23:51,281 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:23:51,282 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T02:23:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:23:51,287 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:23:51,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-09T02:23:51,290 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=135, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:23:51,294 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:51,297 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A, FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B, FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C, FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/recovered.edits] 2024-11-09T02:23:51,300 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/651be8d91a0d40d988b5c52d9ded93ed to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/651be8d91a0d40d988b5c52d9ded93ed 2024-11-09T02:23:51,301 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/69b9e4e5d3a6407598fe75ca83df9948 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/69b9e4e5d3a6407598fe75ca83df9948 2024-11-09T02:23:51,303 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/7c0b186b8a674e0eb40659277eb6f9b5 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/A/7c0b186b8a674e0eb40659277eb6f9b5 2024-11-09T02:23:51,305 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/8a778dd0367442e88e65aea1dc74ca55 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/8a778dd0367442e88e65aea1dc74ca55 2024-11-09T02:23:51,306 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/e4586501584d48da8dd0b866cd6c1c34 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/e4586501584d48da8dd0b866cd6c1c34 2024-11-09T02:23:51,307 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/fcccbde373a442d380a8bb7801837b7c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/B/fcccbde373a442d380a8bb7801837b7c 2024-11-09T02:23:51,311 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/0f8f2203b5f74f808a68c6b923d81fc1 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/0f8f2203b5f74f808a68c6b923d81fc1 2024-11-09T02:23:51,312 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/60313ac4054f4f1e8e9d12937870f19a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/60313ac4054f4f1e8e9d12937870f19a 2024-11-09T02:23:51,313 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/c051cdd5edb44d65b5f92be0ff96b9d5 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/C/c051cdd5edb44d65b5f92be0ff96b9d5 2024-11-09T02:23:51,320 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/recovered.edits/333.seqid to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059/recovered.edits/333.seqid 2024-11-09T02:23:51,320 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/509cbfd6eaffef996b82cfb91370c059 2024-11-09T02:23:51,320 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T02:23:51,322 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=135, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:23:51,324 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T02:23:51,326 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T02:23:51,327 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=135, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:23:51,327 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T02:23:51,327 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731119031327"}]},"ts":"9223372036854775807"} 2024-11-09T02:23:51,328 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T02:23:51,328 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 509cbfd6eaffef996b82cfb91370c059, NAME => 'TestAcidGuarantees,,1731119001397.509cbfd6eaffef996b82cfb91370c059.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T02:23:51,329 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T02:23:51,329 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731119031329"}]},"ts":"9223372036854775807"} 2024-11-09T02:23:51,330 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T02:23:51,331 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=135, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:23:51,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 49 msec 2024-11-09T02:23:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-09T02:23:51,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T02:23:51,406 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:23:51,417 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238 (was 242), OpenFileDescriptor=447 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=403 (was 411), ProcessCount=11 (was 11), AvailableMemoryMB=2880 (was 2885) 2024-11-09T02:23:51,427 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=238, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=403, ProcessCount=11, AvailableMemoryMB=2880 2024-11-09T02:23:51,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T02:23:51,429 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:23:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T02:23:51,431 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:23:51,431 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:51,431 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 136 2024-11-09T02:23:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-09T02:23:51,432 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:23:51,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742323_1499 (size=959) 2024-11-09T02:23:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-09T02:23:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-09T02:23:51,844 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a 2024-11-09T02:23:51,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742324_1500 (size=53) 2024-11-09T02:23:52,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-09T02:23:52,253 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:23:52,253 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 64fc150c6b20bf9e198fa51cd5e2ea2a, disabling compactions & flushes 2024-11-09T02:23:52,253 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:52,253 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:52,254 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. after waiting 0 ms 2024-11-09T02:23:52,254 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:52,254 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:52,254 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: Waiting for close lock at 1731119032253Disabling compacts and flushes for region at 1731119032253Disabling writes for close at 1731119032254 (+1 ms)Writing region close event to WAL at 1731119032254Closed at 1731119032254 2024-11-09T02:23:52,255 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:23:52,255 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731119032255"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731119032255"}]},"ts":"1731119032255"} 2024-11-09T02:23:52,257 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T02:23:52,258 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:23:52,258 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731119032258"}]},"ts":"1731119032258"} 2024-11-09T02:23:52,259 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T02:23:52,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=64fc150c6b20bf9e198fa51cd5e2ea2a, ASSIGN}] 2024-11-09T02:23:52,261 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=64fc150c6b20bf9e198fa51cd5e2ea2a, ASSIGN 2024-11-09T02:23:52,262 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=64fc150c6b20bf9e198fa51cd5e2ea2a, ASSIGN; state=OFFLINE, location=53f286d3af3b,46079,1731118875223; forceNewPlan=false, retain=false 2024-11-09T02:23:52,413 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=64fc150c6b20bf9e198fa51cd5e2ea2a, regionState=OPENING, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:52,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=64fc150c6b20bf9e198fa51cd5e2ea2a, ASSIGN because future has completed 2024-11-09T02:23:52,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223}] 2024-11-09T02:23:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-09T02:23:52,575 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:52,576 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7752): Opening region: {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:23:52,576 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:52,576 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:23:52,576 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7794): checking encryption for 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:52,576 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7797): checking classloading for 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:52,578 INFO [StoreOpener-64fc150c6b20bf9e198fa51cd5e2ea2a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:52,581 INFO [StoreOpener-64fc150c6b20bf9e198fa51cd5e2ea2a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64fc150c6b20bf9e198fa51cd5e2ea2a columnFamilyName A 2024-11-09T02:23:52,581 DEBUG [StoreOpener-64fc150c6b20bf9e198fa51cd5e2ea2a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:52,582 INFO [StoreOpener-64fc150c6b20bf9e198fa51cd5e2ea2a-1 {}] regionserver.HStore(327): Store=64fc150c6b20bf9e198fa51cd5e2ea2a/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:23:52,582 INFO [StoreOpener-64fc150c6b20bf9e198fa51cd5e2ea2a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:52,584 INFO [StoreOpener-64fc150c6b20bf9e198fa51cd5e2ea2a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64fc150c6b20bf9e198fa51cd5e2ea2a columnFamilyName B 2024-11-09T02:23:52,584 DEBUG [StoreOpener-64fc150c6b20bf9e198fa51cd5e2ea2a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:52,585 INFO [StoreOpener-64fc150c6b20bf9e198fa51cd5e2ea2a-1 {}] regionserver.HStore(327): Store=64fc150c6b20bf9e198fa51cd5e2ea2a/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:23:52,585 INFO [StoreOpener-64fc150c6b20bf9e198fa51cd5e2ea2a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:52,587 INFO [StoreOpener-64fc150c6b20bf9e198fa51cd5e2ea2a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64fc150c6b20bf9e198fa51cd5e2ea2a columnFamilyName C 2024-11-09T02:23:52,587 DEBUG [StoreOpener-64fc150c6b20bf9e198fa51cd5e2ea2a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:52,588 INFO [StoreOpener-64fc150c6b20bf9e198fa51cd5e2ea2a-1 {}] regionserver.HStore(327): Store=64fc150c6b20bf9e198fa51cd5e2ea2a/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:23:52,588 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1038): replaying wal for 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:52,589 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:52,589 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:52,590 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1048): stopping wal replay for 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:52,590 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1060): Cleaning up temporary data for 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:52,590 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T02:23:52,591 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1093): writing seq id for 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:52,593 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:23:52,594 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1114): Opened 64fc150c6b20bf9e198fa51cd5e2ea2a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61126471, jitterRate=-0.08914460241794586}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T02:23:52,594 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:52,594 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1006): Region open journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: Running coprocessor pre-open hook at 1731119032576Writing region info on filesystem at 1731119032576Initializing all the Stores at 1731119032578 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731119032578Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731119032578Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731119032578Cleaning up temporary data from old regions at 1731119032590 (+12 ms)Running coprocessor post-open hooks at 1731119032594 (+4 ms)Region opened successfully at 1731119032594 2024-11-09T02:23:52,595 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., pid=138, masterSystemTime=1731119032572 2024-11-09T02:23:52,597 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:52,597 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:52,597 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=64fc150c6b20bf9e198fa51cd5e2ea2a, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:52,599 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 because future has completed 2024-11-09T02:23:52,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-09T02:23:52,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure 64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 in 182 msec 2024-11-09T02:23:52,602 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-09T02:23:52,602 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=64fc150c6b20bf9e198fa51cd5e2ea2a, ASSIGN in 341 msec 2024-11-09T02:23:52,603 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:23:52,603 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731119032603"}]},"ts":"1731119032603"} 2024-11-09T02:23:52,604 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T02:23:52,605 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:23:52,606 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1760 sec 2024-11-09T02:23:53,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-09T02:23:53,581 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:23:53,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e2cba0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:53,584 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:23:53,584 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:23:53,586 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:23:53,587 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:23:53,587 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:23:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@330c90b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:23:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:23:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:53,590 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:23:53,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d0756b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:53,591 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:23:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:23:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:23:53,593 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:23:53,593 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T02:23:53,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 53f286d3af3b,45041,1731118874457 2024-11-09T02:23:53,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1e976af1 2024-11-09T02:23:53,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T02:23:53,595 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T02:23:53,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T02:23:53,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:23:53,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-09T02:23:53,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742325_1501 (size=995) 2024-11-09T02:23:54,013 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-09T02:23:54,013 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-09T02:23:54,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T02:23:54,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=64fc150c6b20bf9e198fa51cd5e2ea2a, REOPEN/MOVE}] 2024-11-09T02:23:54,023 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=64fc150c6b20bf9e198fa51cd5e2ea2a, REOPEN/MOVE 2024-11-09T02:23:54,024 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=64fc150c6b20bf9e198fa51cd5e2ea2a, regionState=CLOSING, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:54,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=64fc150c6b20bf9e198fa51cd5e2ea2a, REOPEN/MOVE because future has completed 2024-11-09T02:23:54,025 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T02:23:54,025 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223}] 2024-11-09T02:23:54,179 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:54,179 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T02:23:54,179 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing 64fc150c6b20bf9e198fa51cd5e2ea2a, disabling compactions & flushes 2024-11-09T02:23:54,179 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:54,179 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:54,179 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. after waiting 0 ms 2024-11-09T02:23:54,179 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:54,188 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:23:54,190 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:54,190 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: Waiting for close lock at 1731119034179Running coprocessor pre-close hooks at 1731119034179Disabling compacts and flushes for region at 1731119034179Disabling writes for close at 1731119034179Writing region close event to WAL at 1731119034181 (+2 ms)Running coprocessor post-close hooks at 1731119034189 (+8 ms)Closed at 1731119034189 2024-11-09T02:23:54,190 WARN [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionServer(3299): Not adding moved region record: 64fc150c6b20bf9e198fa51cd5e2ea2a to self. 2024-11-09T02:23:54,193 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:54,193 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=64fc150c6b20bf9e198fa51cd5e2ea2a, regionState=CLOSED 2024-11-09T02:23:54,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 because future has completed 2024-11-09T02:23:54,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-09T02:23:54,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure 64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 in 171 msec 2024-11-09T02:23:54,199 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=64fc150c6b20bf9e198fa51cd5e2ea2a, REOPEN/MOVE; state=CLOSED, location=53f286d3af3b,46079,1731118875223; forceNewPlan=false, retain=true 2024-11-09T02:23:54,350 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=64fc150c6b20bf9e198fa51cd5e2ea2a, regionState=OPENING, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:54,353 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=64fc150c6b20bf9e198fa51cd5e2ea2a, REOPEN/MOVE because future has completed 2024-11-09T02:23:54,354 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223}] 2024-11-09T02:23:54,601 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:54,601 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7752): Opening region: {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:23:54,602 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:54,602 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:23:54,602 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7794): checking encryption for 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:54,602 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7797): checking classloading for 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:54,605 INFO [StoreOpener-64fc150c6b20bf9e198fa51cd5e2ea2a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:54,607 INFO [StoreOpener-64fc150c6b20bf9e198fa51cd5e2ea2a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64fc150c6b20bf9e198fa51cd5e2ea2a columnFamilyName A 2024-11-09T02:23:54,609 DEBUG [StoreOpener-64fc150c6b20bf9e198fa51cd5e2ea2a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:54,610 INFO [StoreOpener-64fc150c6b20bf9e198fa51cd5e2ea2a-1 {}] regionserver.HStore(327): Store=64fc150c6b20bf9e198fa51cd5e2ea2a/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:23:54,610 INFO [StoreOpener-64fc150c6b20bf9e198fa51cd5e2ea2a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:54,611 INFO [StoreOpener-64fc150c6b20bf9e198fa51cd5e2ea2a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64fc150c6b20bf9e198fa51cd5e2ea2a columnFamilyName B 2024-11-09T02:23:54,611 DEBUG [StoreOpener-64fc150c6b20bf9e198fa51cd5e2ea2a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:54,611 INFO [StoreOpener-64fc150c6b20bf9e198fa51cd5e2ea2a-1 {}] regionserver.HStore(327): Store=64fc150c6b20bf9e198fa51cd5e2ea2a/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:23:54,612 INFO [StoreOpener-64fc150c6b20bf9e198fa51cd5e2ea2a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:54,612 INFO [StoreOpener-64fc150c6b20bf9e198fa51cd5e2ea2a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64fc150c6b20bf9e198fa51cd5e2ea2a columnFamilyName C 2024-11-09T02:23:54,612 DEBUG [StoreOpener-64fc150c6b20bf9e198fa51cd5e2ea2a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:54,613 INFO [StoreOpener-64fc150c6b20bf9e198fa51cd5e2ea2a-1 {}] regionserver.HStore(327): Store=64fc150c6b20bf9e198fa51cd5e2ea2a/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:23:54,613 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1038): replaying wal for 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:54,614 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:54,615 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:54,616 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1048): stopping wal replay for 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:54,617 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1060): Cleaning up temporary data for 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:54,617 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T02:23:54,619 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1093): writing seq id for 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:54,619 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1114): Opened 64fc150c6b20bf9e198fa51cd5e2ea2a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59313676, jitterRate=-0.11615735292434692}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T02:23:54,620 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:54,620 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1006): Region open journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: Running coprocessor pre-open hook at 1731119034602Writing region info on filesystem at 1731119034602Initializing all the Stores at 1731119034604 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731119034604Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731119034605 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731119034605Cleaning up temporary data from old regions at 1731119034617 (+12 ms)Running coprocessor post-open hooks at 1731119034620 (+3 ms)Region opened successfully at 1731119034620 2024-11-09T02:23:54,621 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., pid=143, masterSystemTime=1731119034595 2024-11-09T02:23:54,623 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:54,623 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:54,624 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=64fc150c6b20bf9e198fa51cd5e2ea2a, regionState=OPEN, openSeqNum=5, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:54,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 because future has completed 2024-11-09T02:23:54,629 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=141 2024-11-09T02:23:54,629 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure 64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 in 273 msec 2024-11-09T02:23:54,632 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-09T02:23:54,632 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=64fc150c6b20bf9e198fa51cd5e2ea2a, REOPEN/MOVE in 607 msec 2024-11-09T02:23:54,635 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-09T02:23:54,635 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 612 msec 2024-11-09T02:23:54,638 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0380 sec 2024-11-09T02:23:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-09T02:23:54,640 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:23:54,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744b0f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,641 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:23:54,641 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:23:54,642 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:23:54,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:23:54,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:23:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77d48b77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:23:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:23:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:54,644 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:23:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2540b78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,645 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:23:54,645 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:23:54,646 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:23:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:23:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:23:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7541a014, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:23:54,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:23:54,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:54,647 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:23:54,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43d613c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,648 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:23:54,649 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:23:54,649 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:23:54,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:23:54,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:23:54,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4478d69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:23:54,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:23:54,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:54,651 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:23:54,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35ffd6b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,652 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,652 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:23:54,652 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:23:54,653 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:23:54,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:23:54,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:23:54,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@190ca49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:23:54,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:23:54,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:54,654 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:23:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18142e93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47173f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,655 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:23:54,656 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:23:54,656 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:23:54,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:23:54,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:23:54,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5018a80e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:23:54,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:23:54,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:54,657 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:23:54,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11aef1b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,658 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:23:54,658 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:23:54,659 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:23:54,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:23:54,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:23:54,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dece2ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:23:54,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:23:54,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:54,660 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:23:54,660 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b5aa14a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,661 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:23:54,661 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:23:54,661 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:23:54,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:23:54,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:23:54,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e743554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:23:54,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:23:54,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:54,662 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:23:54,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b0f8b05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,663 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:23:54,663 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:23:54,664 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:23:54,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:23:54,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:23:54,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ffa68d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:23:54,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:23:54,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:54,665 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:23:54,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4226737a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:23:54,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:23:54,666 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:23:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:23:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:23:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50b761ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:23:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:23:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:54,667 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:23:54,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43e1973, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2455dde6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,669 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,45041,-1 for getting cluster id 2024-11-09T02:23:54,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:23:54,669 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ede6ca3-e8ea-4612-bac6-a05540447f39' 2024-11-09T02:23:54,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:23:54,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ede6ca3-e8ea-4612-bac6-a05540447f39" 2024-11-09T02:23:54,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19759277, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,45041,-1] 2024-11-09T02:23:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:23:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:23:54,670 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:23:54,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@89955df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:23:54,671 DEBUG [Thread-2271 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:23:54,671 DEBUG [Thread-2254 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:23:54,671 DEBUG [Thread-2269 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:23:54,671 DEBUG [Thread-2258 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:23:54,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:23:54,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:23:54,672 DEBUG [Thread-2262 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:23:54,672 DEBUG [Thread-2273 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:23:54,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:23:54,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:23:54,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:23:54,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:23:54,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:23:54,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:23:54,673 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:23:54,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:23:54,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:23:54,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:23:54,674 DEBUG [Thread-2260 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:23:54,674 DEBUG [Thread-2265 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:23:54,674 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:23:54,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:23:54,674 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:23:54,674 DEBUG [Thread-2267 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:23:54,674 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:23:54,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:23:54,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:23:54,675 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:23:54,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:23:54,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:23:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-09T02:23:54,675 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:23:54,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:23:54,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:23:54,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:23:54,676 DEBUG [Thread-2256 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:23:54,676 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:23:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-09T02:23:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,46079,1731118875223, seqNum=-1] 2024-11-09T02:23:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:23:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:23:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:23:54,677 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:23:54,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:23:54,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:23:54,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:23:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:23:54,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:23:54,678 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:23:54,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:23:54,678 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:23:54,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:23:54,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:23:54,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:23:54,681 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:23:54,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5] 2024-11-09T02:23:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:54,685 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:23:54,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64fc150c6b20bf9e198fa51cd5e2ea2a 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T02:23:54,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:54,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119094705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119094705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:54,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:54,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:54,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:54,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:54,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:54,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:54,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119094706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:54,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119094707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:54,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119094707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:54,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:54,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:54,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:54,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:54,711 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:54,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:54,711 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:54,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:54,711 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:54,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097e02b375a09741bb9690b42c56461237_64fc150c6b20bf9e198fa51cd5e2ea2a is 50, key is test_row_0/A:col10/1731119034685/Put/seqid=0 2024-11-09T02:23:54,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742326_1502 (size=12154) 2024-11-09T02:23:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-09T02:23:54,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:54,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119094810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119094810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:54,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119094820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:54,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119094820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:54,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119094820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:54,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-09T02:23:54,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:54,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:23:54,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:54,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:54,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:54,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-09T02:23:54,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:54,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:23:54,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:54,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:54,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-09T02:23:55,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119095020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:55,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119095020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:55,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:55,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:55,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119095029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:55,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119095029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119095029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:55,121 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:55,123 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097e02b375a09741bb9690b42c56461237_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097e02b375a09741bb9690b42c56461237_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:55,124 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/7d5c9fb774db4fabb48be03251cd9247, store: [table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:23:55,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/7d5c9fb774db4fabb48be03251cd9247 is 175, key is test_row_0/A:col10/1731119034685/Put/seqid=0 2024-11-09T02:23:55,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742327_1503 (size=30955) 2024-11-09T02:23:55,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-09T02:23:55,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:55,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:23:55,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:55,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:55,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:55,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-09T02:23:55,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:55,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:23:55,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:55,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:55,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:55,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:55,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-09T02:23:55,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119095330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:55,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:55,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119095330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:55,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:55,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119095340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:55,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119095340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:55,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:55,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119095340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:55,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:55,342 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,342 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,342 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:55,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-09T02:23:55,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:55,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:23:55,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:55,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:55,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:55,528 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/7d5c9fb774db4fabb48be03251cd9247 2024-11-09T02:23:55,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/56207b358da44633981e980148f73f21 is 50, key is test_row_0/B:col10/1731119034685/Put/seqid=0 2024-11-09T02:23:55,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742328_1504 (size=12001) 2024-11-09T02:23:55,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-09T02:23:55,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:55,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:23:55,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:55,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:55,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:55,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-09T02:23:55,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:55,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:23:55,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:55,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:55,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-09T02:23:55,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119095840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:55,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:55,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119095840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:55,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:55,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119095849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:55,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:55,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119095849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:55,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:55,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119095849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:55,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:55,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:55,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-09T02:23:55,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:55,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:23:55,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:55,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:55,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:23:55,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/56207b358da44633981e980148f73f21 2024-11-09T02:23:55,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/ae6ed4017c324bafaacad2d4b667ad69 is 50, key is test_row_0/C:col10/1731119034685/Put/seqid=0 2024-11-09T02:23:55,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742329_1505 (size=12001) 2024-11-09T02:23:55,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/ae6ed4017c324bafaacad2d4b667ad69 2024-11-09T02:23:55,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/7d5c9fb774db4fabb48be03251cd9247 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/7d5c9fb774db4fabb48be03251cd9247 2024-11-09T02:23:55,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/7d5c9fb774db4fabb48be03251cd9247, entries=150, sequenceid=16, filesize=30.2 K 2024-11-09T02:23:55,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/56207b358da44633981e980148f73f21 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/56207b358da44633981e980148f73f21 2024-11-09T02:23:55,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/56207b358da44633981e980148f73f21, entries=150, sequenceid=16, filesize=11.7 K 2024-11-09T02:23:55,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/ae6ed4017c324bafaacad2d4b667ad69 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/ae6ed4017c324bafaacad2d4b667ad69 2024-11-09T02:23:55,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/ae6ed4017c324bafaacad2d4b667ad69, entries=150, sequenceid=16, filesize=11.7 K 2024-11-09T02:23:55,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 64fc150c6b20bf9e198fa51cd5e2ea2a in 1304ms, sequenceid=16, compaction requested=false 2024-11-09T02:23:55,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:23:56,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-09T02:23:56,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:56,050 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 64fc150c6b20bf9e198fa51cd5e2ea2a 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-09T02:23:56,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091f873a3d71a348f7a09536fc98bed344_64fc150c6b20bf9e198fa51cd5e2ea2a is 50, key is test_row_0/A:col10/1731119034706/Put/seqid=0 2024-11-09T02:23:56,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742330_1506 (size=12154) 2024-11-09T02:23:56,161 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:23:56,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:56,463 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091f873a3d71a348f7a09536fc98bed344_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091f873a3d71a348f7a09536fc98bed344_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:56,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/cd5539ae0560446f9e39dfd9ea4293c9, store: [table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:23:56,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/cd5539ae0560446f9e39dfd9ea4293c9 is 175, key is test_row_0/A:col10/1731119034706/Put/seqid=0 2024-11-09T02:23:56,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742331_1507 (size=30955) 2024-11-09T02:23:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-09T02:23:56,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:23:56,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:56,869 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/cd5539ae0560446f9e39dfd9ea4293c9 2024-11-09T02:23:56,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/26777c5fb00749bda90b15c59e2243fb is 50, key is test_row_0/B:col10/1731119034706/Put/seqid=0 2024-11-09T02:23:56,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742332_1508 (size=12001) 2024-11-09T02:23:56,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119096872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:56,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119096891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:56,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119096891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:56,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119096891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:56,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:56,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119096895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:56,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:56,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:56,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:57,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119097000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:57,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:57,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:57,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119097000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119097000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:57,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:57,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119097000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:57,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:57,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119097000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:57,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:57,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:57,003 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,003 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,003 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:57,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119097210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:57,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:57,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119097210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:57,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119097210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:57,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:57,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119097210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119097210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:57,276 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/26777c5fb00749bda90b15c59e2243fb 2024-11-09T02:23:57,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/f988146a9870442a84efeeda3169ad7c is 50, key is test_row_0/C:col10/1731119034706/Put/seqid=0 2024-11-09T02:23:57,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742333_1509 (size=12001) 2024-11-09T02:23:57,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119097520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:57,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119097520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:57,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:57,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119097520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:57,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:57,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:57,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119097520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:57,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:57,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119097520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:57,686 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/f988146a9870442a84efeeda3169ad7c 2024-11-09T02:23:57,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/cd5539ae0560446f9e39dfd9ea4293c9 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/cd5539ae0560446f9e39dfd9ea4293c9 2024-11-09T02:23:57,692 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/cd5539ae0560446f9e39dfd9ea4293c9, entries=150, sequenceid=40, filesize=30.2 K 2024-11-09T02:23:57,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/26777c5fb00749bda90b15c59e2243fb as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/26777c5fb00749bda90b15c59e2243fb 2024-11-09T02:23:57,695 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/26777c5fb00749bda90b15c59e2243fb, entries=150, sequenceid=40, filesize=11.7 K 2024-11-09T02:23:57,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/f988146a9870442a84efeeda3169ad7c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/f988146a9870442a84efeeda3169ad7c 2024-11-09T02:23:57,699 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/f988146a9870442a84efeeda3169ad7c, entries=150, sequenceid=40, filesize=11.7 K 2024-11-09T02:23:57,700 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 64fc150c6b20bf9e198fa51cd5e2ea2a in 1649ms, sequenceid=40, compaction requested=false 2024-11-09T02:23:57,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:23:57,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:57,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-09T02:23:57,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-09T02:23:57,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-09T02:23:57,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0240 sec 2024-11-09T02:23:57,704 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 3.0300 sec 2024-11-09T02:23:58,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:58,032 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:23:58,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64fc150c6b20bf9e198fa51cd5e2ea2a 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-09T02:23:58,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109900e0d5957f34f18a23499ec76cd1de6_64fc150c6b20bf9e198fa51cd5e2ea2a is 50, key is test_row_0/A:col10/1731119038032/Put/seqid=0 2024-11-09T02:23:58,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742334_1510 (size=14594) 2024-11-09T02:23:58,042 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:58,046 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109900e0d5957f34f18a23499ec76cd1de6_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109900e0d5957f34f18a23499ec76cd1de6_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:58,047 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/0c9f8d212eb240f284991a073823d67c, store: [table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:23:58,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/0c9f8d212eb240f284991a073823d67c is 175, key is test_row_0/A:col10/1731119038032/Put/seqid=0 2024-11-09T02:23:58,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742335_1511 (size=39549) 2024-11-09T02:23:58,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:58,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119098053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:58,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:58,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:58,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119098054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:58,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:58,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:58,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119098056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:58,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119098057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:58,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:58,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119098058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:58,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:58,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119098169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:58,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119098169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:58,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119098169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:58,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119098170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:58,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:58,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119098170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:58,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119098380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:58,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:58,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119098380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:58,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:58,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:58,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:58,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119098380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:58,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:58,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:58,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119098381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:58,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:58,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119098381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:58,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:58,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:58,453 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/0c9f8d212eb240f284991a073823d67c 2024-11-09T02:23:58,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/4f9edf3edcdd41b9861b7e4c878c52e0 is 50, key is test_row_0/B:col10/1731119038032/Put/seqid=0 2024-11-09T02:23:58,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742336_1512 (size=12001) 2024-11-09T02:23:58,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:58,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119098690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:58,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:58,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119098690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:58,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:58,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:58,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:58,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119098690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:58,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:58,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119098690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:58,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:58,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119098690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:58,694 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,694 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,694 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:58,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:58,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:58,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-09T02:23:58,840 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:23:58,840 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:23:58,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-09T02:23:58,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T02:23:58,842 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:23:58,842 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:23:58,842 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:23:58,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/4f9edf3edcdd41b9861b7e4c878c52e0 2024-11-09T02:23:58,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/17c8bb6e7c5b42c1a2c417717c2c44b0 is 50, key is test_row_0/C:col10/1731119038032/Put/seqid=0 2024-11-09T02:23:58,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742337_1513 (size=12001) 2024-11-09T02:23:58,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/17c8bb6e7c5b42c1a2c417717c2c44b0 2024-11-09T02:23:58,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/0c9f8d212eb240f284991a073823d67c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/0c9f8d212eb240f284991a073823d67c 2024-11-09T02:23:58,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/0c9f8d212eb240f284991a073823d67c, entries=200, sequenceid=56, filesize=38.6 K 2024-11-09T02:23:58,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/4f9edf3edcdd41b9861b7e4c878c52e0 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/4f9edf3edcdd41b9861b7e4c878c52e0 2024-11-09T02:23:58,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/4f9edf3edcdd41b9861b7e4c878c52e0, entries=150, sequenceid=56, filesize=11.7 K 2024-11-09T02:23:58,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/17c8bb6e7c5b42c1a2c417717c2c44b0 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/17c8bb6e7c5b42c1a2c417717c2c44b0 2024-11-09T02:23:58,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/17c8bb6e7c5b42c1a2c417717c2c44b0, entries=150, sequenceid=56, filesize=11.7 K 2024-11-09T02:23:58,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 64fc150c6b20bf9e198fa51cd5e2ea2a in 862ms, sequenceid=56, compaction requested=true 2024-11-09T02:23:58,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:23:58,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64fc150c6b20bf9e198fa51cd5e2ea2a:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:23:58,895 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:58,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:58,895 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:58,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64fc150c6b20bf9e198fa51cd5e2ea2a:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:23:58,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:58,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64fc150c6b20bf9e198fa51cd5e2ea2a:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:23:58,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:58,895 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:58,895 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): 64fc150c6b20bf9e198fa51cd5e2ea2a/A is initiating minor compaction (all files) 2024-11-09T02:23:58,896 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64fc150c6b20bf9e198fa51cd5e2ea2a/A in TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:58,896 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/7d5c9fb774db4fabb48be03251cd9247, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/cd5539ae0560446f9e39dfd9ea4293c9, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/0c9f8d212eb240f284991a073823d67c] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp, totalSize=99.1 K 2024-11-09T02:23:58,896 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:58,896 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. files: [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/7d5c9fb774db4fabb48be03251cd9247, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/cd5539ae0560446f9e39dfd9ea4293c9, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/0c9f8d212eb240f284991a073823d67c] 2024-11-09T02:23:58,896 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:58,896 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d5c9fb774db4fabb48be03251cd9247, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731119034682 2024-11-09T02:23:58,896 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 64fc150c6b20bf9e198fa51cd5e2ea2a/B is initiating minor compaction (all files) 2024-11-09T02:23:58,896 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64fc150c6b20bf9e198fa51cd5e2ea2a/B in TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:58,896 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/56207b358da44633981e980148f73f21, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/26777c5fb00749bda90b15c59e2243fb, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/4f9edf3edcdd41b9861b7e4c878c52e0] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp, totalSize=35.2 K 2024-11-09T02:23:58,896 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd5539ae0560446f9e39dfd9ea4293c9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731119034704 2024-11-09T02:23:58,897 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 56207b358da44633981e980148f73f21, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731119034682 2024-11-09T02:23:58,897 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c9f8d212eb240f284991a073823d67c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731119038031 2024-11-09T02:23:58,897 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 26777c5fb00749bda90b15c59e2243fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731119034704 2024-11-09T02:23:58,897 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f9edf3edcdd41b9861b7e4c878c52e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731119038031 2024-11-09T02:23:58,902 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:23:58,905 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64fc150c6b20bf9e198fa51cd5e2ea2a#B#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:58,905 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/8a43b68dedb44765bf182779a1b2f638 is 50, key is test_row_0/B:col10/1731119038032/Put/seqid=0 2024-11-09T02:23:58,907 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411091ff376dae85d4fc99092bef45cce294f_64fc150c6b20bf9e198fa51cd5e2ea2a store=[table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:23:58,909 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411091ff376dae85d4fc99092bef45cce294f_64fc150c6b20bf9e198fa51cd5e2ea2a, store=[table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:23:58,909 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091ff376dae85d4fc99092bef45cce294f_64fc150c6b20bf9e198fa51cd5e2ea2a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:23:58,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742338_1514 (size=12104) 2024-11-09T02:23:58,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742339_1515 (size=4469) 2024-11-09T02:23:58,942 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64fc150c6b20bf9e198fa51cd5e2ea2a#A#compaction#432 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:58,943 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/ff3ba5a95af843df8df552adf7bc458c is 175, key is test_row_0/A:col10/1731119038032/Put/seqid=0 2024-11-09T02:23:58,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742340_1516 (size=31058) 2024-11-09T02:23:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T02:23:58,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-09T02:23:58,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:58,995 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 64fc150c6b20bf9e198fa51cd5e2ea2a 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-09T02:23:59,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ebadd40418214691ab0638f2780bc196_64fc150c6b20bf9e198fa51cd5e2ea2a is 50, key is test_row_0/A:col10/1731119038056/Put/seqid=0 2024-11-09T02:23:59,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742341_1517 (size=12154) 2024-11-09T02:23:59,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T02:23:59,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:59,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:23:59,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119099210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:59,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:59,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:59,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119099214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:59,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:59,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119099216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:59,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:59,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119099218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:59,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119099218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:59,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:59,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119099320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:59,322 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:59,328 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/8a43b68dedb44765bf182779a1b2f638 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/8a43b68dedb44765bf182779a1b2f638 2024-11-09T02:23:59,332 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64fc150c6b20bf9e198fa51cd5e2ea2a/B of 64fc150c6b20bf9e198fa51cd5e2ea2a into 8a43b68dedb44765bf182779a1b2f638(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:59,332 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:23:59,332 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., storeName=64fc150c6b20bf9e198fa51cd5e2ea2a/B, priority=13, startTime=1731119038895; duration=0sec 2024-11-09T02:23:59,332 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:23:59,332 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64fc150c6b20bf9e198fa51cd5e2ea2a:B 2024-11-09T02:23:59,332 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:23:59,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119099330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:59,333 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:23:59,333 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 64fc150c6b20bf9e198fa51cd5e2ea2a/C is initiating minor compaction (all files) 2024-11-09T02:23:59,333 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64fc150c6b20bf9e198fa51cd5e2ea2a/C in TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:23:59,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:59,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,333 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/ae6ed4017c324bafaacad2d4b667ad69, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/f988146a9870442a84efeeda3169ad7c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/17c8bb6e7c5b42c1a2c417717c2c44b0] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp, totalSize=35.2 K 2024-11-09T02:23:59,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119099330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:59,334 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting ae6ed4017c324bafaacad2d4b667ad69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731119034682 2024-11-09T02:23:59,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:59,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:59,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119099330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:59,334 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting f988146a9870442a84efeeda3169ad7c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731119034704 2024-11-09T02:23:59,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:59,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:59,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119099331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:59,334 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 17c8bb6e7c5b42c1a2c417717c2c44b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731119038031 2024-11-09T02:23:59,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:59,340 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64fc150c6b20bf9e198fa51cd5e2ea2a#C#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:23:59,340 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/45a9c6f06e234ab18d8539a52706a3e9 is 50, key is test_row_0/C:col10/1731119038032/Put/seqid=0 2024-11-09T02:23:59,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742342_1518 (size=12104) 2024-11-09T02:23:59,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:23:59,421 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/ff3ba5a95af843df8df552adf7bc458c as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/ff3ba5a95af843df8df552adf7bc458c 2024-11-09T02:23:59,422 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ebadd40418214691ab0638f2780bc196_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ebadd40418214691ab0638f2780bc196_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:23:59,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/cbffb8aece114e9c9a07e2fbb78c76a0, store: [table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:23:59,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/cbffb8aece114e9c9a07e2fbb78c76a0 is 175, key is test_row_0/A:col10/1731119038056/Put/seqid=0 2024-11-09T02:23:59,425 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64fc150c6b20bf9e198fa51cd5e2ea2a/A of 64fc150c6b20bf9e198fa51cd5e2ea2a into ff3ba5a95af843df8df552adf7bc458c(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:59,425 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:23:59,425 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., storeName=64fc150c6b20bf9e198fa51cd5e2ea2a/A, priority=13, startTime=1731119038895; duration=0sec 2024-11-09T02:23:59,425 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:59,425 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64fc150c6b20bf9e198fa51cd5e2ea2a:A 2024-11-09T02:23:59,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742343_1519 (size=30955) 2024-11-09T02:23:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T02:23:59,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:59,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119099530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:59,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:59,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119099540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:59,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:59,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119099540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:59,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119099540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:59,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:59,543 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:59,543 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,543 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119099540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:59,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:59,751 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/45a9c6f06e234ab18d8539a52706a3e9 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/45a9c6f06e234ab18d8539a52706a3e9 2024-11-09T02:23:59,756 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64fc150c6b20bf9e198fa51cd5e2ea2a/C of 64fc150c6b20bf9e198fa51cd5e2ea2a into 45a9c6f06e234ab18d8539a52706a3e9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:23:59,757 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:23:59,757 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., storeName=64fc150c6b20bf9e198fa51cd5e2ea2a/C, priority=13, startTime=1731119038895; duration=0sec 2024-11-09T02:23:59,757 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:23:59,757 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64fc150c6b20bf9e198fa51cd5e2ea2a:C 2024-11-09T02:23:59,827 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/cbffb8aece114e9c9a07e2fbb78c76a0 2024-11-09T02:23:59,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/cb76a7f13a8b4bcfbe8fa8acbbb163e0 is 50, key is test_row_0/B:col10/1731119038056/Put/seqid=0 2024-11-09T02:23:59,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742344_1520 (size=12001) 2024-11-09T02:23:59,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119099840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:59,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:59,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119099850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:59,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:59,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:59,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119099850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:59,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119099850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:59,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:59,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:23:59,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119099850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:23:59,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:59,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:23:59,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:23:59,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T02:24:00,237 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/cb76a7f13a8b4bcfbe8fa8acbbb163e0 2024-11-09T02:24:00,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/5407eba030344e30be69f3545f174bf2 is 50, key is test_row_0/C:col10/1731119038056/Put/seqid=0 2024-11-09T02:24:00,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742345_1521 (size=12001) 2024-11-09T02:24:00,247 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/5407eba030344e30be69f3545f174bf2 2024-11-09T02:24:00,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/cbffb8aece114e9c9a07e2fbb78c76a0 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/cbffb8aece114e9c9a07e2fbb78c76a0 2024-11-09T02:24:00,253 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/cbffb8aece114e9c9a07e2fbb78c76a0, entries=150, sequenceid=76, filesize=30.2 K 2024-11-09T02:24:00,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/cb76a7f13a8b4bcfbe8fa8acbbb163e0 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/cb76a7f13a8b4bcfbe8fa8acbbb163e0 2024-11-09T02:24:00,256 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/cb76a7f13a8b4bcfbe8fa8acbbb163e0, entries=150, sequenceid=76, filesize=11.7 K 2024-11-09T02:24:00,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/5407eba030344e30be69f3545f174bf2 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/5407eba030344e30be69f3545f174bf2 2024-11-09T02:24:00,258 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/5407eba030344e30be69f3545f174bf2, entries=150, sequenceid=76, filesize=11.7 K 2024-11-09T02:24:00,259 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 64fc150c6b20bf9e198fa51cd5e2ea2a in 1265ms, sequenceid=76, compaction requested=false 2024-11-09T02:24:00,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:24:00,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:00,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-09T02:24:00,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-09T02:24:00,262 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-09T02:24:00,262 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4180 sec 2024-11-09T02:24:00,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.4220 sec 2024-11-09T02:24:00,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:00,352 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:24:00,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64fc150c6b20bf9e198fa51cd5e2ea2a 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-09T02:24:00,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110936ed756c4dc54886b8602585815b97ee_64fc150c6b20bf9e198fa51cd5e2ea2a is 50, key is test_row_0/A:col10/1731119039213/Put/seqid=0 2024-11-09T02:24:00,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742346_1522 (size=12154) 2024-11-09T02:24:00,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:00,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119100398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:00,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:00,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:00,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119100398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:00,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:00,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:00,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119100398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:00,405 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,405 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,405 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:00,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:00,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119100399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:00,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:00,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:00,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119100399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:00,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:00,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:00,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119100509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:00,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:00,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119100510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119100510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119100510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119100510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:00,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119100720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:00,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119100720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:00,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119100720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119100720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:00,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119100720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:00,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:00,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:00,761 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:24:00,764 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110936ed756c4dc54886b8602585815b97ee_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110936ed756c4dc54886b8602585815b97ee_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:00,764 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/7b9e245db2154eeebf307bf3daf0c831, store: [table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:00,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/7b9e245db2154eeebf307bf3daf0c831 is 175, key is test_row_0/A:col10/1731119039213/Put/seqid=0 2024-11-09T02:24:00,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742347_1523 (size=30955) 2024-11-09T02:24:00,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T02:24:00,990 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:24:00,991 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:24:00,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-09T02:24:00,992 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:24:00,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T02:24:00,992 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:24:00,993 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:24:01,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119101029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:01,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:01,031 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:01,031 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119101030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:01,031 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:01,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119101030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:01,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119101030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:01,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119101030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:01,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T02:24:01,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-09T02:24:01,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:01,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:24:01,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:01,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:01,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:01,168 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/7b9e245db2154eeebf307bf3daf0c831 2024-11-09T02:24:01,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/91d6cb4b545d4d9a8645fd491b0784f4 is 50, key is test_row_0/B:col10/1731119039213/Put/seqid=0 2024-11-09T02:24:01,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742348_1524 (size=12001) 2024-11-09T02:24:01,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/91d6cb4b545d4d9a8645fd491b0784f4 2024-11-09T02:24:01,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/353906c611e4482983c4fffc8d865e0b is 50, key is test_row_0/C:col10/1731119039213/Put/seqid=0 2024-11-09T02:24:01,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742349_1525 (size=12001) 2024-11-09T02:24:01,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-09T02:24:01,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:01,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:24:01,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:01,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:01,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T02:24:01,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-09T02:24:01,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:01,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:24:01,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:01,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:01,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:01,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119101540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:01,543 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:01,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:01,543 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:01,543 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119101540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:01,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:01,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:01,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119101540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:01,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:01,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:01,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:01,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:01,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119101540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:01,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:01,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:01,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:01,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119101540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:01,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:01,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:01,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:01,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/353906c611e4482983c4fffc8d865e0b 2024-11-09T02:24:01,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/7b9e245db2154eeebf307bf3daf0c831 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/7b9e245db2154eeebf307bf3daf0c831 2024-11-09T02:24:01,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-09T02:24:01,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:01,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:24:01,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:01,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:01,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:01,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/7b9e245db2154eeebf307bf3daf0c831, entries=150, sequenceid=97, filesize=30.2 K 2024-11-09T02:24:01,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/91d6cb4b545d4d9a8645fd491b0784f4 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/91d6cb4b545d4d9a8645fd491b0784f4 2024-11-09T02:24:01,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/91d6cb4b545d4d9a8645fd491b0784f4, entries=150, sequenceid=97, filesize=11.7 K 2024-11-09T02:24:01,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/353906c611e4482983c4fffc8d865e0b as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/353906c611e4482983c4fffc8d865e0b 2024-11-09T02:24:01,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/353906c611e4482983c4fffc8d865e0b, entries=150, sequenceid=97, filesize=11.7 K 2024-11-09T02:24:01,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 64fc150c6b20bf9e198fa51cd5e2ea2a in 1264ms, sequenceid=97, compaction requested=true 2024-11-09T02:24:01,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:24:01,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64fc150c6b20bf9e198fa51cd5e2ea2a:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:24:01,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:24:01,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64fc150c6b20bf9e198fa51cd5e2ea2a:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:24:01,617 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:24:01,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:24:01,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64fc150c6b20bf9e198fa51cd5e2ea2a:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:24:01,617 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:24:01,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:24:01,618 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:24:01,618 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:24:01,618 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): 64fc150c6b20bf9e198fa51cd5e2ea2a/A is initiating minor compaction (all files) 2024-11-09T02:24:01,618 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 64fc150c6b20bf9e198fa51cd5e2ea2a/B is initiating minor compaction (all files) 2024-11-09T02:24:01,618 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64fc150c6b20bf9e198fa51cd5e2ea2a/B in TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:01,618 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/8a43b68dedb44765bf182779a1b2f638, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/cb76a7f13a8b4bcfbe8fa8acbbb163e0, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/91d6cb4b545d4d9a8645fd491b0784f4] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp, totalSize=35.3 K 2024-11-09T02:24:01,618 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64fc150c6b20bf9e198fa51cd5e2ea2a/A in TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:01,618 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/ff3ba5a95af843df8df552adf7bc458c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/cbffb8aece114e9c9a07e2fbb78c76a0, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/7b9e245db2154eeebf307bf3daf0c831] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp, totalSize=90.8 K 2024-11-09T02:24:01,618 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:01,618 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. files: [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/ff3ba5a95af843df8df552adf7bc458c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/cbffb8aece114e9c9a07e2fbb78c76a0, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/7b9e245db2154eeebf307bf3daf0c831] 2024-11-09T02:24:01,619 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a43b68dedb44765bf182779a1b2f638, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731119038031 2024-11-09T02:24:01,619 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting cb76a7f13a8b4bcfbe8fa8acbbb163e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731119038049 2024-11-09T02:24:01,619 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff3ba5a95af843df8df552adf7bc458c, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731119038031 2024-11-09T02:24:01,619 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 91d6cb4b545d4d9a8645fd491b0784f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731119039213 2024-11-09T02:24:01,619 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbffb8aece114e9c9a07e2fbb78c76a0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731119038049 2024-11-09T02:24:01,620 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b9e245db2154eeebf307bf3daf0c831, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731119039213 2024-11-09T02:24:01,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T02:24:01,625 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64fc150c6b20bf9e198fa51cd5e2ea2a#B#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:24:01,627 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/f16a1bfa3ddb4d6da24a288f526316a3 is 50, key is test_row_0/B:col10/1731119039213/Put/seqid=0 2024-11-09T02:24:01,627 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:01,629 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109d31ca1cab442481aa1ba6e966c3a5762_64fc150c6b20bf9e198fa51cd5e2ea2a store=[table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:01,630 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109d31ca1cab442481aa1ba6e966c3a5762_64fc150c6b20bf9e198fa51cd5e2ea2a, store=[table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:01,631 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d31ca1cab442481aa1ba6e966c3a5762_64fc150c6b20bf9e198fa51cd5e2ea2a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:01,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742350_1526 (size=12207) 2024-11-09T02:24:01,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742351_1527 (size=4469) 2024-11-09T02:24:01,638 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64fc150c6b20bf9e198fa51cd5e2ea2a#A#compaction#442 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:24:01,639 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/314b85e7e50e48f6a99e9185efdff2a2 is 175, key is test_row_0/A:col10/1731119039213/Put/seqid=0 2024-11-09T02:24:01,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742352_1528 (size=31161) 2024-11-09T02:24:01,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-09T02:24:01,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:01,757 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 64fc150c6b20bf9e198fa51cd5e2ea2a 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-09T02:24:01,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109538c4f9612ae4bfcbaafb7142a57a939_64fc150c6b20bf9e198fa51cd5e2ea2a is 50, key is test_row_0/A:col10/1731119040398/Put/seqid=0 2024-11-09T02:24:01,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742353_1529 (size=12154) 2024-11-09T02:24:01,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:24:01,768 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109538c4f9612ae4bfcbaafb7142a57a939_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109538c4f9612ae4bfcbaafb7142a57a939_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:01,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/b11b49ea6ca642e1ba0970c3d759a30d, store: [table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:01,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/b11b49ea6ca642e1ba0970c3d759a30d is 175, key is test_row_0/A:col10/1731119040398/Put/seqid=0 2024-11-09T02:24:01,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742354_1530 (size=30955) 2024-11-09T02:24:02,035 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/f16a1bfa3ddb4d6da24a288f526316a3 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/f16a1bfa3ddb4d6da24a288f526316a3 2024-11-09T02:24:02,038 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64fc150c6b20bf9e198fa51cd5e2ea2a/B of 64fc150c6b20bf9e198fa51cd5e2ea2a into f16a1bfa3ddb4d6da24a288f526316a3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:24:02,038 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:24:02,038 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., storeName=64fc150c6b20bf9e198fa51cd5e2ea2a/B, priority=13, startTime=1731119041617; duration=0sec 2024-11-09T02:24:02,038 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:24:02,038 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64fc150c6b20bf9e198fa51cd5e2ea2a:B 2024-11-09T02:24:02,038 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:24:02,039 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:24:02,039 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 64fc150c6b20bf9e198fa51cd5e2ea2a/C is initiating minor compaction (all files) 2024-11-09T02:24:02,039 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64fc150c6b20bf9e198fa51cd5e2ea2a/C in TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:02,039 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/45a9c6f06e234ab18d8539a52706a3e9, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/5407eba030344e30be69f3545f174bf2, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/353906c611e4482983c4fffc8d865e0b] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp, totalSize=35.3 K 2024-11-09T02:24:02,039 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 45a9c6f06e234ab18d8539a52706a3e9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731119038031 2024-11-09T02:24:02,040 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 5407eba030344e30be69f3545f174bf2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731119038049 2024-11-09T02:24:02,040 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 353906c611e4482983c4fffc8d865e0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731119039213 2024-11-09T02:24:02,045 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/314b85e7e50e48f6a99e9185efdff2a2 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/314b85e7e50e48f6a99e9185efdff2a2 2024-11-09T02:24:02,045 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64fc150c6b20bf9e198fa51cd5e2ea2a#C#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:24:02,046 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/23ee1205664c44598843cdc8ef74e217 is 50, key is test_row_0/C:col10/1731119039213/Put/seqid=0 2024-11-09T02:24:02,049 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64fc150c6b20bf9e198fa51cd5e2ea2a/A of 64fc150c6b20bf9e198fa51cd5e2ea2a into 314b85e7e50e48f6a99e9185efdff2a2(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:24:02,049 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:24:02,049 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., storeName=64fc150c6b20bf9e198fa51cd5e2ea2a/A, priority=13, startTime=1731119041617; duration=0sec 2024-11-09T02:24:02,049 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:24:02,049 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64fc150c6b20bf9e198fa51cd5e2ea2a:A 2024-11-09T02:24:02,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742355_1531 (size=12207) 2024-11-09T02:24:02,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T02:24:02,172 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/b11b49ea6ca642e1ba0970c3d759a30d 2024-11-09T02:24:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/960cf28778af4efbb200a72e201c9a07 is 50, key is test_row_0/B:col10/1731119040398/Put/seqid=0 2024-11-09T02:24:02,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742356_1532 (size=12001) 2024-11-09T02:24:02,457 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/23ee1205664c44598843cdc8ef74e217 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/23ee1205664c44598843cdc8ef74e217 2024-11-09T02:24:02,460 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64fc150c6b20bf9e198fa51cd5e2ea2a/C of 64fc150c6b20bf9e198fa51cd5e2ea2a into 23ee1205664c44598843cdc8ef74e217(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:24:02,460 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:24:02,460 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., storeName=64fc150c6b20bf9e198fa51cd5e2ea2a/C, priority=13, startTime=1731119041617; duration=0sec 2024-11-09T02:24:02,461 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:24:02,461 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64fc150c6b20bf9e198fa51cd5e2ea2a:C 2024-11-09T02:24:02,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:02,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:24:02,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:02,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119102564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:02,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119102564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:02,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:02,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:02,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119102565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:02,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119102566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:02,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119102567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:02,583 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/960cf28778af4efbb200a72e201c9a07 2024-11-09T02:24:02,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/867b7bf2a3494533823d593e1e151cb2 is 50, key is test_row_0/C:col10/1731119040398/Put/seqid=0 2024-11-09T02:24:02,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742357_1533 (size=12001) 2024-11-09T02:24:02,591 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/867b7bf2a3494533823d593e1e151cb2 2024-11-09T02:24:02,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/b11b49ea6ca642e1ba0970c3d759a30d as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/b11b49ea6ca642e1ba0970c3d759a30d 2024-11-09T02:24:02,598 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/b11b49ea6ca642e1ba0970c3d759a30d, entries=150, sequenceid=116, filesize=30.2 K 2024-11-09T02:24:02,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/960cf28778af4efbb200a72e201c9a07 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/960cf28778af4efbb200a72e201c9a07 2024-11-09T02:24:02,601 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/960cf28778af4efbb200a72e201c9a07, entries=150, sequenceid=116, filesize=11.7 K 2024-11-09T02:24:02,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/867b7bf2a3494533823d593e1e151cb2 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/867b7bf2a3494533823d593e1e151cb2 2024-11-09T02:24:02,604 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/867b7bf2a3494533823d593e1e151cb2, entries=150, sequenceid=116, filesize=11.7 K 2024-11-09T02:24:02,605 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 64fc150c6b20bf9e198fa51cd5e2ea2a in 848ms, sequenceid=116, compaction requested=false 2024-11-09T02:24:02,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:24:02,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:02,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-09T02:24:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-09T02:24:02,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-09T02:24:02,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6140 sec 2024-11-09T02:24:02,610 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.6180 sec 2024-11-09T02:24:02,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:02,683 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:24:02,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64fc150c6b20bf9e198fa51cd5e2ea2a 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-09T02:24:02,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110905238632915a41e4be25294df4d21b8d_64fc150c6b20bf9e198fa51cd5e2ea2a is 50, key is test_row_0/A:col10/1731119042559/Put/seqid=0 2024-11-09T02:24:02,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742358_1534 (size=12304) 2024-11-09T02:24:02,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:02,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:02,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119102696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:02,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119102696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:02,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:02,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119102697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:02,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:02,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:02,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:02,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:02,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119102701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:02,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:02,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:02,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119102701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:02,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:02,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:02,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119102810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:02,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:02,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:02,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119102810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:02,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:02,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119102810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:02,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:02,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119102810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:02,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:02,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:02,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119102810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:02,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:02,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:03,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:03,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119103020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:03,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:03,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119103020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:03,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:03,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119103020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:03,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119103020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:03,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119103020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:03,093 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:24:03,096 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110905238632915a41e4be25294df4d21b8d_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110905238632915a41e4be25294df4d21b8d_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:03,097 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/c7d673be91194bfc897533bf154f8292, store: [table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:03,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/c7d673be91194bfc897533bf154f8292 is 175, key is test_row_0/A:col10/1731119042559/Put/seqid=0 2024-11-09T02:24:03,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742359_1535 (size=31105) 2024-11-09T02:24:03,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T02:24:03,140 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:24:03,140 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:24:03,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-09T02:24:03,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T02:24:03,142 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:24:03,142 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:24:03,143 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:24:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T02:24:03,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T02:24:03,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:03,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:24:03,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:03,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:03,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:03,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119103330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:03,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:03,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119103330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:03,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:03,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119103330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:03,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:03,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119103330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:03,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:03,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119103330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:03,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:03,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T02:24:03,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:03,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:24:03,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:03,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:03,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T02:24:03,500 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=138, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/c7d673be91194bfc897533bf154f8292 2024-11-09T02:24:03,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/383d21ee398346ce86506541dbf6193e is 50, key is test_row_0/B:col10/1731119042559/Put/seqid=0 2024-11-09T02:24:03,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742360_1536 (size=12151) 2024-11-09T02:24:03,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T02:24:03,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:03,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:24:03,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:03,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:03,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:03,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:03,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T02:24:03,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:03,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:24:03,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:03,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:03,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:03,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T02:24:03,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:03,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119103840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:03,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119103840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:03,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119103840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:03,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119103841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:03,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119103841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:03,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T02:24:03,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:03,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:24:03,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:03,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:03,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:03,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:03,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/383d21ee398346ce86506541dbf6193e 2024-11-09T02:24:03,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/1edfafd88f29485ca50977c31e81ad15 is 50, key is test_row_0/C:col10/1731119042559/Put/seqid=0 2024-11-09T02:24:03,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742361_1537 (size=12151) 2024-11-09T02:24:04,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T02:24:04,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:04,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:24:04,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:04,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:04,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:04,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:04,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T02:24:04,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:04,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:24:04,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:04,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:04,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:04,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:04,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T02:24:04,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/1edfafd88f29485ca50977c31e81ad15 2024-11-09T02:24:04,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/c7d673be91194bfc897533bf154f8292 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/c7d673be91194bfc897533bf154f8292 2024-11-09T02:24:04,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/c7d673be91194bfc897533bf154f8292, entries=150, sequenceid=138, filesize=30.4 K 2024-11-09T02:24:04,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/383d21ee398346ce86506541dbf6193e as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/383d21ee398346ce86506541dbf6193e 2024-11-09T02:24:04,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/383d21ee398346ce86506541dbf6193e, entries=150, sequenceid=138, filesize=11.9 K 2024-11-09T02:24:04,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/1edfafd88f29485ca50977c31e81ad15 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/1edfafd88f29485ca50977c31e81ad15 2024-11-09T02:24:04,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/1edfafd88f29485ca50977c31e81ad15, entries=150, sequenceid=138, filesize=11.9 K 2024-11-09T02:24:04,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 64fc150c6b20bf9e198fa51cd5e2ea2a in 1647ms, sequenceid=138, compaction requested=true 2024-11-09T02:24:04,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:24:04,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64fc150c6b20bf9e198fa51cd5e2ea2a:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:24:04,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:24:04,330 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:24:04,330 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:24:04,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64fc150c6b20bf9e198fa51cd5e2ea2a:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:24:04,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:24:04,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64fc150c6b20bf9e198fa51cd5e2ea2a:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:24:04,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:24:04,330 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:24:04,330 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:24:04,330 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 64fc150c6b20bf9e198fa51cd5e2ea2a/B is initiating minor compaction (all files) 2024-11-09T02:24:04,330 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): 64fc150c6b20bf9e198fa51cd5e2ea2a/A is initiating minor compaction (all files) 2024-11-09T02:24:04,331 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64fc150c6b20bf9e198fa51cd5e2ea2a/B in TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:04,331 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64fc150c6b20bf9e198fa51cd5e2ea2a/A in TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:04,331 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/f16a1bfa3ddb4d6da24a288f526316a3, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/960cf28778af4efbb200a72e201c9a07, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/383d21ee398346ce86506541dbf6193e] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp, totalSize=35.5 K 2024-11-09T02:24:04,331 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/314b85e7e50e48f6a99e9185efdff2a2, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/b11b49ea6ca642e1ba0970c3d759a30d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/c7d673be91194bfc897533bf154f8292] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp, totalSize=91.0 K 2024-11-09T02:24:04,331 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:04,331 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. files: [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/314b85e7e50e48f6a99e9185efdff2a2, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/b11b49ea6ca642e1ba0970c3d759a30d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/c7d673be91194bfc897533bf154f8292] 2024-11-09T02:24:04,331 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting f16a1bfa3ddb4d6da24a288f526316a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731119039213 2024-11-09T02:24:04,331 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 314b85e7e50e48f6a99e9185efdff2a2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731119039213 2024-11-09T02:24:04,331 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 960cf28778af4efbb200a72e201c9a07, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731119040368 2024-11-09T02:24:04,331 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 383d21ee398346ce86506541dbf6193e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731119042559 2024-11-09T02:24:04,331 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting b11b49ea6ca642e1ba0970c3d759a30d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731119040368 2024-11-09T02:24:04,332 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7d673be91194bfc897533bf154f8292, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731119042559 2024-11-09T02:24:04,336 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:04,337 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64fc150c6b20bf9e198fa51cd5e2ea2a#B#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:24:04,338 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/9be10c39973e440dbc770a47a4210560 is 50, key is test_row_0/B:col10/1731119042559/Put/seqid=0 2024-11-09T02:24:04,338 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411098e262cf196d1461fb93f2a85a286d8be_64fc150c6b20bf9e198fa51cd5e2ea2a store=[table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:04,339 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411098e262cf196d1461fb93f2a85a286d8be_64fc150c6b20bf9e198fa51cd5e2ea2a, store=[table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:04,340 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411098e262cf196d1461fb93f2a85a286d8be_64fc150c6b20bf9e198fa51cd5e2ea2a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:04,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742362_1538 (size=12459) 2024-11-09T02:24:04,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T02:24:04,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:04,362 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 64fc150c6b20bf9e198fa51cd5e2ea2a 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-09T02:24:04,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742363_1539 (size=4469) 2024-11-09T02:24:04,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099deb5f3deb204098a2fcf796a55698dc_64fc150c6b20bf9e198fa51cd5e2ea2a is 50, key is test_row_0/A:col10/1731119042696/Put/seqid=0 2024-11-09T02:24:04,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742364_1540 (size=12304) 2024-11-09T02:24:04,749 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/9be10c39973e440dbc770a47a4210560 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/9be10c39973e440dbc770a47a4210560 2024-11-09T02:24:04,752 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64fc150c6b20bf9e198fa51cd5e2ea2a/B of 64fc150c6b20bf9e198fa51cd5e2ea2a into 9be10c39973e440dbc770a47a4210560(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:24:04,752 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:24:04,752 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., storeName=64fc150c6b20bf9e198fa51cd5e2ea2a/B, priority=13, startTime=1731119044330; duration=0sec 2024-11-09T02:24:04,752 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:24:04,752 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64fc150c6b20bf9e198fa51cd5e2ea2a:B 2024-11-09T02:24:04,753 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:24:04,753 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:24:04,753 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 64fc150c6b20bf9e198fa51cd5e2ea2a/C is initiating minor compaction (all files) 2024-11-09T02:24:04,754 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64fc150c6b20bf9e198fa51cd5e2ea2a/C in TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:04,754 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/23ee1205664c44598843cdc8ef74e217, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/867b7bf2a3494533823d593e1e151cb2, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/1edfafd88f29485ca50977c31e81ad15] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp, totalSize=35.5 K 2024-11-09T02:24:04,754 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 23ee1205664c44598843cdc8ef74e217, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731119039213 2024-11-09T02:24:04,754 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 867b7bf2a3494533823d593e1e151cb2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731119040368 2024-11-09T02:24:04,754 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 1edfafd88f29485ca50977c31e81ad15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731119042559 2024-11-09T02:24:04,760 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64fc150c6b20bf9e198fa51cd5e2ea2a#C#compaction#453 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:24:04,760 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/1c7180544d2b4e8887387b0548bd4590 is 50, key is test_row_0/C:col10/1731119042559/Put/seqid=0 2024-11-09T02:24:04,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742365_1541 (size=12459) 2024-11-09T02:24:04,766 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/1c7180544d2b4e8887387b0548bd4590 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/1c7180544d2b4e8887387b0548bd4590 2024-11-09T02:24:04,770 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64fc150c6b20bf9e198fa51cd5e2ea2a/C of 64fc150c6b20bf9e198fa51cd5e2ea2a into 1c7180544d2b4e8887387b0548bd4590(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:24:04,770 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:24:04,770 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., storeName=64fc150c6b20bf9e198fa51cd5e2ea2a/C, priority=13, startTime=1731119044330; duration=0sec 2024-11-09T02:24:04,770 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:24:04,770 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64fc150c6b20bf9e198fa51cd5e2ea2a:C 2024-11-09T02:24:04,790 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64fc150c6b20bf9e198fa51cd5e2ea2a#A#compaction#451 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:24:04,791 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/4ef5b9b8380a4629a6a57d52bb0052ab is 175, key is test_row_0/A:col10/1731119042559/Put/seqid=0 2024-11-09T02:24:04,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742366_1542 (size=31413) 2024-11-09T02:24:04,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:24:04,799 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099deb5f3deb204098a2fcf796a55698dc_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099deb5f3deb204098a2fcf796a55698dc_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:04,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/a8c095d3bfa34139b73be89b7cb23ead, store: [table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:04,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/a8c095d3bfa34139b73be89b7cb23ead is 175, key is test_row_0/A:col10/1731119042696/Put/seqid=0 2024-11-09T02:24:04,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742367_1543 (size=31105) 2024-11-09T02:24:04,803 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/a8c095d3bfa34139b73be89b7cb23ead 2024-11-09T02:24:04,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/b6194983120145ff82809595d41b5570 is 50, key is test_row_0/B:col10/1731119042696/Put/seqid=0 2024-11-09T02:24:04,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742368_1544 (size=12151) 2024-11-09T02:24:04,811 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/b6194983120145ff82809595d41b5570 2024-11-09T02:24:04,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/76cc6c04ada74a8abcc6929fb69ee507 is 50, key is test_row_0/C:col10/1731119042696/Put/seqid=0 2024-11-09T02:24:04,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742369_1545 (size=12151) 2024-11-09T02:24:04,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:04,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:24:04,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:04,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119104864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:04,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:04,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:04,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:04,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:04,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119104866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:04,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:04,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:04,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:04,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:04,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119104867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:04,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:04,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:04,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:04,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:04,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119104868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:04,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:04,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:04,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:04,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:04,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119104868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:04,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:04,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:04,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:04,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:04,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119104979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:04,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:04,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:04,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119104979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:04,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:04,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:04,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:04,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:04,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:04,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:04,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119104980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:04,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:04,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:04,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:04,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:04,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119104980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:04,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:04,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:04,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:04,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:04,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119104980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:04,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:04,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:04,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:05,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119105189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:05,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:05,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119105189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:05,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119105190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:05,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:05,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119105190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:05,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:05,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119105190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:05,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:05,193 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,193 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,193 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:05,197 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/4ef5b9b8380a4629a6a57d52bb0052ab as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/4ef5b9b8380a4629a6a57d52bb0052ab 2024-11-09T02:24:05,200 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64fc150c6b20bf9e198fa51cd5e2ea2a/A of 64fc150c6b20bf9e198fa51cd5e2ea2a into 4ef5b9b8380a4629a6a57d52bb0052ab(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:24:05,201 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:24:05,201 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., storeName=64fc150c6b20bf9e198fa51cd5e2ea2a/A, priority=13, startTime=1731119044330; duration=0sec 2024-11-09T02:24:05,201 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:24:05,201 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64fc150c6b20bf9e198fa51cd5e2ea2a:A 2024-11-09T02:24:05,219 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/76cc6c04ada74a8abcc6929fb69ee507 2024-11-09T02:24:05,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/a8c095d3bfa34139b73be89b7cb23ead as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/a8c095d3bfa34139b73be89b7cb23ead 2024-11-09T02:24:05,225 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/a8c095d3bfa34139b73be89b7cb23ead, entries=150, sequenceid=156, filesize=30.4 K 2024-11-09T02:24:05,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/b6194983120145ff82809595d41b5570 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/b6194983120145ff82809595d41b5570 2024-11-09T02:24:05,229 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/b6194983120145ff82809595d41b5570, entries=150, sequenceid=156, filesize=11.9 K 2024-11-09T02:24:05,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/76cc6c04ada74a8abcc6929fb69ee507 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/76cc6c04ada74a8abcc6929fb69ee507 2024-11-09T02:24:05,232 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/76cc6c04ada74a8abcc6929fb69ee507, entries=150, sequenceid=156, filesize=11.9 K 2024-11-09T02:24:05,233 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 64fc150c6b20bf9e198fa51cd5e2ea2a in 871ms, sequenceid=156, compaction requested=false 2024-11-09T02:24:05,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:24:05,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:05,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-09T02:24:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-09T02:24:05,235 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-09T02:24:05,235 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0910 sec 2024-11-09T02:24:05,237 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.0960 sec 2024-11-09T02:24:05,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T02:24:05,290 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:24:05,290 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:24:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-09T02:24:05,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-09T02:24:05,292 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:24:05,292 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:24:05,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:24:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-09T02:24:05,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-09T02:24:05,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:05,445 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 64fc150c6b20bf9e198fa51cd5e2ea2a 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-09T02:24:05,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109352575967ad54b01a410c406d4d2021a_64fc150c6b20bf9e198fa51cd5e2ea2a is 50, key is test_row_0/A:col10/1731119044867/Put/seqid=0 2024-11-09T02:24:05,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742370_1546 (size=12304) 2024-11-09T02:24:05,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:05,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:24:05,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:05,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119105511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:05,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:05,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:05,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119105512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:05,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:05,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:05,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119105514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:05,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:05,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:05,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119105515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,521 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119105516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:05,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-09T02:24:05,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119105620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:05,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119105620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:05,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:05,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119105630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:05,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119105630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:05,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:05,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:05,633 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,633 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119105630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:05,633 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:05,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:05,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119105829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:05,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119105830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:05,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:05,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119105840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:05,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:05,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:05,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119105840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:05,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:05,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119105840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:05,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:24:05,856 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109352575967ad54b01a410c406d4d2021a_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109352575967ad54b01a410c406d4d2021a_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:05,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/d95ce1e76195434e9cadc281c950cf93, store: [table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:05,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/d95ce1e76195434e9cadc281c950cf93 is 175, key is test_row_0/A:col10/1731119044867/Put/seqid=0 2024-11-09T02:24:05,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742371_1547 (size=31105) 2024-11-09T02:24:05,860 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=178, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/d95ce1e76195434e9cadc281c950cf93 2024-11-09T02:24:05,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/b0fcfe8cb6414c72ad6c243686d0ee0e is 50, key is test_row_0/B:col10/1731119044867/Put/seqid=0 2024-11-09T02:24:05,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742372_1548 (size=12151) 2024-11-09T02:24:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-09T02:24:06,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119106139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119106139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:06,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119106150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:06,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:06,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119106150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:06,152 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:06,152 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,152 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119106150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:06,278 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/b0fcfe8cb6414c72ad6c243686d0ee0e 2024-11-09T02:24:06,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/2b13ffac795a4992849a8654d936db67 is 50, key is test_row_0/C:col10/1731119044867/Put/seqid=0 2024-11-09T02:24:06,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742373_1549 (size=12151) 2024-11-09T02:24:06,288 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/2b13ffac795a4992849a8654d936db67 2024-11-09T02:24:06,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/d95ce1e76195434e9cadc281c950cf93 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/d95ce1e76195434e9cadc281c950cf93 2024-11-09T02:24:06,294 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/d95ce1e76195434e9cadc281c950cf93, entries=150, sequenceid=178, filesize=30.4 K 2024-11-09T02:24:06,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/b0fcfe8cb6414c72ad6c243686d0ee0e as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/b0fcfe8cb6414c72ad6c243686d0ee0e 2024-11-09T02:24:06,297 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/b0fcfe8cb6414c72ad6c243686d0ee0e, entries=150, sequenceid=178, filesize=11.9 K 2024-11-09T02:24:06,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/2b13ffac795a4992849a8654d936db67 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/2b13ffac795a4992849a8654d936db67 2024-11-09T02:24:06,305 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/2b13ffac795a4992849a8654d936db67, entries=150, sequenceid=178, filesize=11.9 K 2024-11-09T02:24:06,306 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 64fc150c6b20bf9e198fa51cd5e2ea2a in 861ms, sequenceid=178, compaction requested=true 2024-11-09T02:24:06,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:24:06,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:06,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-09T02:24:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-09T02:24:06,309 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-09T02:24:06,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0150 sec 2024-11-09T02:24:06,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.0200 sec 2024-11-09T02:24:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-09T02:24:06,430 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:24:06,431 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:24:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-09T02:24:06,432 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:24:06,433 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:24:06,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:24:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T02:24:06,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T02:24:06,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-09T02:24:06,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:06,585 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 64fc150c6b20bf9e198fa51cd5e2ea2a 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-09T02:24:06,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110985363a86ae884eb8b2d19ea444f9984d_64fc150c6b20bf9e198fa51cd5e2ea2a is 50, key is test_row_0/A:col10/1731119045514/Put/seqid=0 2024-11-09T02:24:06,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742374_1550 (size=12304) 2024-11-09T02:24:06,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:24:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:06,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119106670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:06,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119106670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:06,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119106670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119106672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:06,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:06,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119106672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:06,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:06,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T02:24:06,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:06,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119106780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:06,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119106780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:06,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119106780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119106781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:06,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:06,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119106790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:06,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:06,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119106990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:24:06,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119106990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119106990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:06,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119106991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:06,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:06,996 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110985363a86ae884eb8b2d19ea444f9984d_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110985363a86ae884eb8b2d19ea444f9984d_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:06,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/eed9a67bd18948569cbd67f91fe403b6, store: [table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:06,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/eed9a67bd18948569cbd67f91fe403b6 is 175, key is test_row_0/A:col10/1731119045514/Put/seqid=0 2024-11-09T02:24:07,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742375_1551 (size=31105) 2024-11-09T02:24:07,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119107000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:07,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:07,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:07,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T02:24:07,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119107299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:07,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119107299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:07,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:07,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119107300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:07,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:07,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119107300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:07,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119107310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:07,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:07,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:07,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:07,401 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/eed9a67bd18948569cbd67f91fe403b6 2024-11-09T02:24:07,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/5dada534fae243a58c1b2d59861f5ed4 is 50, key is test_row_0/B:col10/1731119045514/Put/seqid=0 2024-11-09T02:24:07,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742376_1552 (size=12151) 2024-11-09T02:24:07,409 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/5dada534fae243a58c1b2d59861f5ed4 2024-11-09T02:24:07,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/5c92c0e862fc45ec913ffc2ec1691362 is 50, key is test_row_0/C:col10/1731119045514/Put/seqid=0 2024-11-09T02:24:07,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742377_1553 (size=12151) 2024-11-09T02:24:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T02:24:07,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:07,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119107809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:07,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:07,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119107809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:07,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:07,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:07,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:07,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:07,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:07,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:07,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:07,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119107810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:07,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:07,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:07,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:07,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:07,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119107810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:07,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:07,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:07,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:07,819 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/5c92c0e862fc45ec913ffc2ec1691362 2024-11-09T02:24:07,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/eed9a67bd18948569cbd67f91fe403b6 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/eed9a67bd18948569cbd67f91fe403b6 2024-11-09T02:24:07,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:07,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119107820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:07,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:07,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:07,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:07,824 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/eed9a67bd18948569cbd67f91fe403b6, entries=150, sequenceid=196, filesize=30.4 K 2024-11-09T02:24:07,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/5dada534fae243a58c1b2d59861f5ed4 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/5dada534fae243a58c1b2d59861f5ed4 2024-11-09T02:24:07,827 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/5dada534fae243a58c1b2d59861f5ed4, entries=150, sequenceid=196, filesize=11.9 K 2024-11-09T02:24:07,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/5c92c0e862fc45ec913ffc2ec1691362 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/5c92c0e862fc45ec913ffc2ec1691362 2024-11-09T02:24:07,830 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/5c92c0e862fc45ec913ffc2ec1691362, entries=150, sequenceid=196, filesize=11.9 K 2024-11-09T02:24:07,830 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 64fc150c6b20bf9e198fa51cd5e2ea2a in 1245ms, sequenceid=196, compaction requested=true 2024-11-09T02:24:07,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:24:07,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:07,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-09T02:24:07,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-09T02:24:07,833 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-09T02:24:07,833 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3980 sec 2024-11-09T02:24:07,835 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.4020 sec 2024-11-09T02:24:08,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T02:24:08,580 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:24:08,580 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:24:08,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-09T02:24:08,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T02:24:08,582 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:24:08,582 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:24:08,582 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:24:08,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T02:24:08,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-09T02:24:08,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:08,734 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 64fc150c6b20bf9e198fa51cd5e2ea2a 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-09T02:24:08,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110973169e5b5b554a2c8c3ae6eaec7d245d_64fc150c6b20bf9e198fa51cd5e2ea2a is 50, key is test_row_0/A:col10/1731119046669/Put/seqid=0 2024-11-09T02:24:08,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742378_1554 (size=12304) 2024-11-09T02:24:08,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:24:08,746 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110973169e5b5b554a2c8c3ae6eaec7d245d_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110973169e5b5b554a2c8c3ae6eaec7d245d_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:08,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/8bbf3108fc2f43cf9b19a91af9536d11, store: [table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:08,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/8bbf3108fc2f43cf9b19a91af9536d11 is 175, key is test_row_0/A:col10/1731119046669/Put/seqid=0 2024-11-09T02:24:08,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742379_1555 (size=31105) 2024-11-09T02:24:08,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:08,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:24:08,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:08,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119108839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119108840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:08,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119108840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:08,846 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_1' in 64fc150c6b20bf9e198fa51cd5e2ea2a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:24:08,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:08,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:08,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:08,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:08,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119108844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:08,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:08,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:08,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:08,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:08,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119108846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:08,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:08,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:08,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T02:24:08,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119108950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:08,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119108950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:08,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119108960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:08,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:08,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:08,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:08,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119108960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:08,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:08,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:08,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:09,161 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/8bbf3108fc2f43cf9b19a91af9536d11 2024-11-09T02:24:09,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:09,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119109160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:09,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:09,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:09,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:09,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:09,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119109160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:09,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:09,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:09,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:09,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/ab5f122e499c4fee9148c71705b0256a is 50, key is test_row_0/B:col10/1731119046669/Put/seqid=0 2024-11-09T02:24:09,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742380_1556 (size=12151) 2024-11-09T02:24:09,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:09,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119109170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:09,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:09,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:09,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:09,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:09,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119109170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:09,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:09,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:09,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T02:24:09,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119109470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:09,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:09,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:09,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:09,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119109470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:09,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:09,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:09,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:09,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119109480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:09,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:09,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:09,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:09,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119109480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:09,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:09,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:09,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:09,570 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/ab5f122e499c4fee9148c71705b0256a 2024-11-09T02:24:09,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/bb202127b82b44cb84d6b53caba11b98 is 50, key is test_row_0/C:col10/1731119046669/Put/seqid=0 2024-11-09T02:24:09,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742381_1557 (size=12151) 2024-11-09T02:24:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T02:24:09,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:09,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119109979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:09,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:09,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:09,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:09,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:09,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119109980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:09,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:09,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:09,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:09,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:09,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119109990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:09,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:09,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:09,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:09,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119109990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:09,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:09,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:09,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:09,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:10,026 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/bb202127b82b44cb84d6b53caba11b98 2024-11-09T02:24:10,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/8bbf3108fc2f43cf9b19a91af9536d11 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/8bbf3108fc2f43cf9b19a91af9536d11 2024-11-09T02:24:10,032 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/8bbf3108fc2f43cf9b19a91af9536d11, entries=150, sequenceid=214, filesize=30.4 K 2024-11-09T02:24:10,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/ab5f122e499c4fee9148c71705b0256a as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/ab5f122e499c4fee9148c71705b0256a 2024-11-09T02:24:10,035 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/ab5f122e499c4fee9148c71705b0256a, entries=150, sequenceid=214, filesize=11.9 K 2024-11-09T02:24:10,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/bb202127b82b44cb84d6b53caba11b98 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/bb202127b82b44cb84d6b53caba11b98 2024-11-09T02:24:10,039 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/bb202127b82b44cb84d6b53caba11b98, entries=150, sequenceid=214, filesize=11.9 K 2024-11-09T02:24:10,039 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 64fc150c6b20bf9e198fa51cd5e2ea2a in 1305ms, sequenceid=214, compaction requested=true 2024-11-09T02:24:10,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:24:10,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:10,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-09T02:24:10,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-09T02:24:10,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-09T02:24:10,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4590 sec 2024-11-09T02:24:10,043 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.4620 sec 2024-11-09T02:24:10,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T02:24:10,730 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:24:10,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T02:24:10,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-09T02:24:10,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T02:24:10,732 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T02:24:10,733 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T02:24:10,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T02:24:10,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T02:24:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:10,863 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:24:10,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64fc150c6b20bf9e198fa51cd5e2ea2a 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-09T02:24:10,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095f4869eca00140349bae8b47f18600b7_64fc150c6b20bf9e198fa51cd5e2ea2a is 50, key is test_row_0/A:col10/1731119048845/Put/seqid=0 2024-11-09T02:24:10,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742382_1558 (size=14794) 2024-11-09T02:24:10,876 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:24:10,880 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095f4869eca00140349bae8b47f18600b7_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095f4869eca00140349bae8b47f18600b7_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:10,880 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/aac24c7335704b85b7d4cb19f5bdcd26, store: [table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:10,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/aac24c7335704b85b7d4cb19f5bdcd26 is 175, key is test_row_0/A:col10/1731119048845/Put/seqid=0 2024-11-09T02:24:10,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742383_1559 (size=39749) 2024-11-09T02:24:10,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-09T02:24:10,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:10,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:24:10,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:10,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:10,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:10,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:10,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119110933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:10,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:10,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119110990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:10,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:10,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119110990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:10,994 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_0' in 64fc150c6b20bf9e198fa51cd5e2ea2a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:24:10,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:10,994 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_2' in 64fc150c6b20bf9e198fa51cd5e2ea2a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:24:10,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:10,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:10,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:10,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:10,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:11,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119110999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:11,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119111000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:11,007 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_1' in 64fc150c6b20bf9e198fa51cd5e2ea2a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:24:11,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:11,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:11,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:11,007 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_0' in 64fc150c6b20bf9e198fa51cd5e2ea2a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:24:11,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:11,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:11,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:11,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-09T02:24:11,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:11,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:24:11,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:11,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:11,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:11,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119111040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:11,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:11,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:11,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T02:24:11,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-09T02:24:11,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:11,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:24:11,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:11,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:11,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:11,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:11,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119111250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:11,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:11,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:11,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:11,284 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/aac24c7335704b85b7d4cb19f5bdcd26 2024-11-09T02:24:11,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/0fbe81ca519c4e50853f06068d674863 is 50, key is test_row_0/B:col10/1731119048845/Put/seqid=0 2024-11-09T02:24:11,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742384_1560 (size=12151) 2024-11-09T02:24:11,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-09T02:24:11,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:11,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:24:11,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:11,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:11,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:11,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:11,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T02:24:11,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-09T02:24:11,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:11,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:24:11,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:11,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:11,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:11,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:11,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:11,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119111560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:11,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:11,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:11,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:11,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-09T02:24:11,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:11,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:24:11,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:11,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:11,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:11,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/0fbe81ca519c4e50853f06068d674863 2024-11-09T02:24:11,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/b795f648fd5a4430ae419b9218fd6066 is 50, key is test_row_0/C:col10/1731119048845/Put/seqid=0 2024-11-09T02:24:11,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742385_1561 (size=12151) 2024-11-09T02:24:11,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-09T02:24:11,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:11,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:24:11,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:11,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:11,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T02:24:11,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-09T02:24:11,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:11,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:24:11,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:11,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:11,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:11,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:12,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119112070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:12,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:12,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:12,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:12,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/b795f648fd5a4430ae419b9218fd6066 2024-11-09T02:24:12,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-09T02:24:12,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:12,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:24:12,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:12,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:12,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:12,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/aac24c7335704b85b7d4cb19f5bdcd26 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/aac24c7335704b85b7d4cb19f5bdcd26 2024-11-09T02:24:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:24:12,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/aac24c7335704b85b7d4cb19f5bdcd26, entries=200, sequenceid=233, filesize=38.8 K 2024-11-09T02:24:12,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/0fbe81ca519c4e50853f06068d674863 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/0fbe81ca519c4e50853f06068d674863 2024-11-09T02:24:12,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/0fbe81ca519c4e50853f06068d674863, entries=150, sequenceid=233, filesize=11.9 K 2024-11-09T02:24:12,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/b795f648fd5a4430ae419b9218fd6066 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/b795f648fd5a4430ae419b9218fd6066 2024-11-09T02:24:12,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/b795f648fd5a4430ae419b9218fd6066, entries=150, sequenceid=233, filesize=11.9 K 2024-11-09T02:24:12,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 64fc150c6b20bf9e198fa51cd5e2ea2a in 1253ms, sequenceid=233, compaction requested=true 2024-11-09T02:24:12,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:24:12,116 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-09T02:24:12,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64fc150c6b20bf9e198fa51cd5e2ea2a:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:24:12,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:24:12,116 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-09T02:24:12,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64fc150c6b20bf9e198fa51cd5e2ea2a:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:24:12,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:24:12,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64fc150c6b20bf9e198fa51cd5e2ea2a:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:24:12,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:24:12,117 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 195582 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-09T02:24:12,117 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73214 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-09T02:24:12,118 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 64fc150c6b20bf9e198fa51cd5e2ea2a/B is initiating minor compaction (all files) 2024-11-09T02:24:12,118 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): 64fc150c6b20bf9e198fa51cd5e2ea2a/A is initiating minor compaction (all files) 2024-11-09T02:24:12,118 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64fc150c6b20bf9e198fa51cd5e2ea2a/B in TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:12,118 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/9be10c39973e440dbc770a47a4210560, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/b6194983120145ff82809595d41b5570, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/b0fcfe8cb6414c72ad6c243686d0ee0e, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/5dada534fae243a58c1b2d59861f5ed4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/ab5f122e499c4fee9148c71705b0256a, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/0fbe81ca519c4e50853f06068d674863] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp, totalSize=71.5 K 2024-11-09T02:24:12,118 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64fc150c6b20bf9e198fa51cd5e2ea2a/A in TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:12,118 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/4ef5b9b8380a4629a6a57d52bb0052ab, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/a8c095d3bfa34139b73be89b7cb23ead, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/d95ce1e76195434e9cadc281c950cf93, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/eed9a67bd18948569cbd67f91fe403b6, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/8bbf3108fc2f43cf9b19a91af9536d11, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/aac24c7335704b85b7d4cb19f5bdcd26] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp, totalSize=191.0 K 2024-11-09T02:24:12,118 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:12,118 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 9be10c39973e440dbc770a47a4210560, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731119042559 2024-11-09T02:24:12,118 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. files: [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/4ef5b9b8380a4629a6a57d52bb0052ab, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/a8c095d3bfa34139b73be89b7cb23ead, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/d95ce1e76195434e9cadc281c950cf93, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/eed9a67bd18948569cbd67f91fe403b6, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/8bbf3108fc2f43cf9b19a91af9536d11, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/aac24c7335704b85b7d4cb19f5bdcd26] 2024-11-09T02:24:12,118 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting b6194983120145ff82809595d41b5570, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731119042688 2024-11-09T02:24:12,118 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ef5b9b8380a4629a6a57d52bb0052ab, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731119042559 2024-11-09T02:24:12,118 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting b0fcfe8cb6414c72ad6c243686d0ee0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731119044863 2024-11-09T02:24:12,118 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8c095d3bfa34139b73be89b7cb23ead, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731119042688 2024-11-09T02:24:12,119 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dada534fae243a58c1b2d59861f5ed4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731119045511 2024-11-09T02:24:12,119 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting d95ce1e76195434e9cadc281c950cf93, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731119044863 2024-11-09T02:24:12,119 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting ab5f122e499c4fee9148c71705b0256a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731119046668 2024-11-09T02:24:12,119 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting eed9a67bd18948569cbd67f91fe403b6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731119045511 2024-11-09T02:24:12,119 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bbf3108fc2f43cf9b19a91af9536d11, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731119046668 2024-11-09T02:24:12,119 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fbe81ca519c4e50853f06068d674863, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731119048843 2024-11-09T02:24:12,119 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting aac24c7335704b85b7d4cb19f5bdcd26, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731119048837 2024-11-09T02:24:12,125 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:12,127 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110999dc958b99864a95b5c2f6c2e62ddbfa_64fc150c6b20bf9e198fa51cd5e2ea2a store=[table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:12,127 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64fc150c6b20bf9e198fa51cd5e2ea2a#B#compaction#468 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:24:12,128 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/bc7a41ca80324944aef0cbe1cd1319a7 is 50, key is test_row_0/B:col10/1731119048845/Put/seqid=0 2024-11-09T02:24:12,130 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110999dc958b99864a95b5c2f6c2e62ddbfa_64fc150c6b20bf9e198fa51cd5e2ea2a, store=[table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:12,130 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110999dc958b99864a95b5c2f6c2e62ddbfa_64fc150c6b20bf9e198fa51cd5e2ea2a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:12,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742386_1562 (size=12663) 2024-11-09T02:24:12,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742387_1563 (size=4469) 2024-11-09T02:24:12,141 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/bc7a41ca80324944aef0cbe1cd1319a7 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/bc7a41ca80324944aef0cbe1cd1319a7 2024-11-09T02:24:12,144 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 64fc150c6b20bf9e198fa51cd5e2ea2a/B of 64fc150c6b20bf9e198fa51cd5e2ea2a into bc7a41ca80324944aef0cbe1cd1319a7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:24:12,144 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:24:12,144 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., storeName=64fc150c6b20bf9e198fa51cd5e2ea2a/B, priority=10, startTime=1731119052116; duration=0sec 2024-11-09T02:24:12,144 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:24:12,144 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64fc150c6b20bf9e198fa51cd5e2ea2a:B 2024-11-09T02:24:12,144 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-09T02:24:12,146 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73214 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-09T02:24:12,146 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 64fc150c6b20bf9e198fa51cd5e2ea2a/C is initiating minor compaction (all files) 2024-11-09T02:24:12,146 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64fc150c6b20bf9e198fa51cd5e2ea2a/C in TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:12,146 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/1c7180544d2b4e8887387b0548bd4590, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/76cc6c04ada74a8abcc6929fb69ee507, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/2b13ffac795a4992849a8654d936db67, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/5c92c0e862fc45ec913ffc2ec1691362, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/bb202127b82b44cb84d6b53caba11b98, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/b795f648fd5a4430ae419b9218fd6066] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp, totalSize=71.5 K 2024-11-09T02:24:12,147 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c7180544d2b4e8887387b0548bd4590, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731119042559 2024-11-09T02:24:12,148 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 76cc6c04ada74a8abcc6929fb69ee507, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731119042688 2024-11-09T02:24:12,148 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b13ffac795a4992849a8654d936db67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731119044863 2024-11-09T02:24:12,148 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c92c0e862fc45ec913ffc2ec1691362, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731119045511 2024-11-09T02:24:12,148 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting bb202127b82b44cb84d6b53caba11b98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731119046668 2024-11-09T02:24:12,148 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting b795f648fd5a4430ae419b9218fd6066, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731119048843 2024-11-09T02:24:12,163 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64fc150c6b20bf9e198fa51cd5e2ea2a#C#compaction#470 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:24:12,163 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/bd617ec810ea48c4b182687a4c229f74 is 50, key is test_row_0/C:col10/1731119048845/Put/seqid=0 2024-11-09T02:24:12,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742388_1564 (size=12663) 2024-11-09T02:24:12,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-09T02:24:12,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:12,257 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 64fc150c6b20bf9e198fa51cd5e2ea2a 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-09T02:24:12,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097fca22ea34714f2e8b35bc27fb7b3a77_64fc150c6b20bf9e198fa51cd5e2ea2a is 50, key is test_row_0/A:col10/1731119050918/Put/seqid=0 2024-11-09T02:24:12,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742389_1565 (size=12304) 2024-11-09T02:24:12,540 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64fc150c6b20bf9e198fa51cd5e2ea2a#A#compaction#469 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:24:12,541 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/b11790976f1f471bb3910198ca1d8673 is 175, key is test_row_0/A:col10/1731119048845/Put/seqid=0 2024-11-09T02:24:12,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742390_1566 (size=31617) 2024-11-09T02:24:12,570 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/bd617ec810ea48c4b182687a4c229f74 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/bd617ec810ea48c4b182687a4c229f74 2024-11-09T02:24:12,573 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 64fc150c6b20bf9e198fa51cd5e2ea2a/C of 64fc150c6b20bf9e198fa51cd5e2ea2a into bd617ec810ea48c4b182687a4c229f74(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:24:12,573 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:24:12,573 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., storeName=64fc150c6b20bf9e198fa51cd5e2ea2a/C, priority=10, startTime=1731119052116; duration=0sec 2024-11-09T02:24:12,573 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:24:12,573 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64fc150c6b20bf9e198fa51cd5e2ea2a:C 2024-11-09T02:24:12,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:24:12,669 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097fca22ea34714f2e8b35bc27fb7b3a77_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097fca22ea34714f2e8b35bc27fb7b3a77_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:12,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/af27a5e199314750a60083857c59c1fb, store: [table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:12,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/af27a5e199314750a60083857c59c1fb is 175, key is test_row_0/A:col10/1731119050918/Put/seqid=0 2024-11-09T02:24:12,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742391_1567 (size=31105) 2024-11-09T02:24:12,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T02:24:12,947 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/b11790976f1f471bb3910198ca1d8673 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/b11790976f1f471bb3910198ca1d8673 2024-11-09T02:24:12,950 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 64fc150c6b20bf9e198fa51cd5e2ea2a/A of 64fc150c6b20bf9e198fa51cd5e2ea2a into b11790976f1f471bb3910198ca1d8673(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:24:12,950 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:24:12,950 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., storeName=64fc150c6b20bf9e198fa51cd5e2ea2a/A, priority=10, startTime=1731119052116; duration=0sec 2024-11-09T02:24:12,950 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:24:12,950 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64fc150c6b20bf9e198fa51cd5e2ea2a:A 2024-11-09T02:24:13,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:13,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. as already flushing 2024-11-09T02:24:13,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:13,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119113033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:13,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:13,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:13,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:13,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119113034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:13,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119113035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:13,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119113035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:13,074 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/af27a5e199314750a60083857c59c1fb 2024-11-09T02:24:13,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/5910788a59c946a5934b05e2d772eafc is 50, key is test_row_0/B:col10/1731119050918/Put/seqid=0 2024-11-09T02:24:13,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:13,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119113080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:13,082 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_2' in 64fc150c6b20bf9e198fa51cd5e2ea2a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:24:13,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:13,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742392_1568 (size=12151) 2024-11-09T02:24:13,083 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/5910788a59c946a5934b05e2d772eafc 2024-11-09T02:24:13,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/3cfc47dd578a486289e93e96c5c01641 is 50, key is test_row_0/C:col10/1731119050918/Put/seqid=0 2024-11-09T02:24:13,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742393_1569 (size=12151) 2024-11-09T02:24:13,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:13,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119113140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:13,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:13,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:13,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119113149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:13,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:13,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:13,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119113149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:13,152 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,152 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:13,152 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:13,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119113149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:13,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:13,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:13,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119113350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:13,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:13,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:13,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119113360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:13,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:13,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:13,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119113360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:13,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:13,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:13,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119113360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:13,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:13,497 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/3cfc47dd578a486289e93e96c5c01641 2024-11-09T02:24:13,499 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T02:24:13,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/af27a5e199314750a60083857c59c1fb as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/af27a5e199314750a60083857c59c1fb 2024-11-09T02:24:13,502 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/af27a5e199314750a60083857c59c1fb, entries=150, sequenceid=251, filesize=30.4 K 2024-11-09T02:24:13,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/5910788a59c946a5934b05e2d772eafc as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/5910788a59c946a5934b05e2d772eafc 2024-11-09T02:24:13,506 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/5910788a59c946a5934b05e2d772eafc, entries=150, sequenceid=251, filesize=11.9 K 2024-11-09T02:24:13,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/3cfc47dd578a486289e93e96c5c01641 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/3cfc47dd578a486289e93e96c5c01641 2024-11-09T02:24:13,510 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/3cfc47dd578a486289e93e96c5c01641, entries=150, sequenceid=251, filesize=11.9 K 2024-11-09T02:24:13,510 INFO [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 64fc150c6b20bf9e198fa51cd5e2ea2a in 1253ms, sequenceid=251, compaction requested=false 2024-11-09T02:24:13,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:24:13,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:13,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-09T02:24:13,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-09T02:24:13,513 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-09T02:24:13,513 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7790 sec 2024-11-09T02:24:13,515 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.7820 sec 2024-11-09T02:24:13,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:13,666 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:24:13,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64fc150c6b20bf9e198fa51cd5e2ea2a 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-09T02:24:13,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109282c7ffc7b084130995081ec43333396_64fc150c6b20bf9e198fa51cd5e2ea2a is 50, key is test_row_0/A:col10/1731119053033/Put/seqid=0 2024-11-09T02:24:13,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742394_1570 (size=14994) 2024-11-09T02:24:13,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:13,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119113683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:13,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:13,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:13,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119113684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:13,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119113684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:13,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:13,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119113685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:13,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:13,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:13,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119113790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:13,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:13,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:13,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119113800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:13,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:13,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119113800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:13,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:13,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119113800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:13,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:13,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:14,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:14,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119114000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:14,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:14,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:14,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:14,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119114010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:14,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:14,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119114010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:14,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:14,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119114010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:14,076 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:24:14,079 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109282c7ffc7b084130995081ec43333396_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109282c7ffc7b084130995081ec43333396_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:14,079 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/2c1d747b316f42e9a8bfd7249ad1375e, store: [table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:14,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/2c1d747b316f42e9a8bfd7249ad1375e is 175, key is test_row_0/A:col10/1731119053033/Put/seqid=0 2024-11-09T02:24:14,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742395_1571 (size=39949) 2024-11-09T02:24:14,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:14,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119114309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:14,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:14,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:14,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:14,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119114320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:14,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119114320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:14,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119114320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:14,483 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/2c1d747b316f42e9a8bfd7249ad1375e 2024-11-09T02:24:14,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/8eca84cbedbe4c0d928797645205f47d is 50, key is test_row_0/B:col10/1731119053033/Put/seqid=0 2024-11-09T02:24:14,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742396_1572 (size=12301) 2024-11-09T02:24:14,675 INFO [Thread-2267 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2267. 2024-11-09T02:24:14,676 DEBUG [Thread-2267 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:24:14,676 DEBUG [Thread-2267 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:14,676 DEBUG [Thread-2267 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:14,676 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:24:14,676 INFO [Thread-2269 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2269. 2024-11-09T02:24:14,676 DEBUG [Thread-2269 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:24:14,676 DEBUG [Thread-2269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:14,677 DEBUG [Thread-2269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:14,677 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:24:14,678 INFO [Thread-2265 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2265. 2024-11-09T02:24:14,678 DEBUG [Thread-2265 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:24:14,678 DEBUG [Thread-2265 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:14,678 DEBUG [Thread-2265 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:14,678 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:24:14,679 INFO [Thread-2271 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2271. 2024-11-09T02:24:14,679 DEBUG [Thread-2271 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:24:14,679 DEBUG [Thread-2271 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:14,679 DEBUG [Thread-2271 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:14,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:24:14,679 INFO [Thread-2273 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2273. 2024-11-09T02:24:14,679 DEBUG [Thread-2273 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:24:14,679 DEBUG [Thread-2273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:14,680 DEBUG [Thread-2273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:14,680 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:24:14,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:14,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60004 deadline: 1731119114820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:14,820 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:14,820 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:14,820 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:14,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:14,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59952 deadline: 1731119114830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:14,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:14,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59992 deadline: 1731119114830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:14,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:14,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:14,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59964 deadline: 1731119114830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:14,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:14,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:14,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:14,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:14,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:14,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:14,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:14,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:14,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/8eca84cbedbe4c0d928797645205f47d 2024-11-09T02:24:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T02:24:14,901 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:24:14,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/47f6709da0d340788d1bf462dd37e744 is 50, key is test_row_0/C:col10/1731119053033/Put/seqid=0 2024-11-09T02:24:14,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742397_1573 (size=12301) 2024-11-09T02:24:15,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:24:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59980 deadline: 1731119115101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:15,103 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 53f286d3af3b,46079,1731118875223 for 'test_row_2' in 64fc150c6b20bf9e198fa51cd5e2ea2a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T02:24:15,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:15,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T02:24:15,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., hostname=53f286d3af3b,46079,1731118875223, seqNum=5 because the exception is null or not the one we care about 2024-11-09T02:24:15,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/47f6709da0d340788d1bf462dd37e744 2024-11-09T02:24:15,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/2c1d747b316f42e9a8bfd7249ad1375e as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/2c1d747b316f42e9a8bfd7249ad1375e 2024-11-09T02:24:15,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/2c1d747b316f42e9a8bfd7249ad1375e, entries=200, sequenceid=273, filesize=39.0 K 2024-11-09T02:24:15,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/8eca84cbedbe4c0d928797645205f47d as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/8eca84cbedbe4c0d928797645205f47d 2024-11-09T02:24:15,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/8eca84cbedbe4c0d928797645205f47d, entries=150, sequenceid=273, filesize=12.0 K 2024-11-09T02:24:15,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/47f6709da0d340788d1bf462dd37e744 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/47f6709da0d340788d1bf462dd37e744 2024-11-09T02:24:15,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/47f6709da0d340788d1bf462dd37e744, entries=150, sequenceid=273, filesize=12.0 K 2024-11-09T02:24:15,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 64fc150c6b20bf9e198fa51cd5e2ea2a in 1665ms, sequenceid=273, compaction requested=true 2024-11-09T02:24:15,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:24:15,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64fc150c6b20bf9e198fa51cd5e2ea2a:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T02:24:15,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:24:15,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64fc150c6b20bf9e198fa51cd5e2ea2a:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T02:24:15,332 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:24:15,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:24:15,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64fc150c6b20bf9e198fa51cd5e2ea2a:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T02:24:15,332 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:24:15,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:24:15,333 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102671 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:24:15,333 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:24:15,333 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1541): 64fc150c6b20bf9e198fa51cd5e2ea2a/A is initiating minor compaction (all files) 2024-11-09T02:24:15,333 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 64fc150c6b20bf9e198fa51cd5e2ea2a/B is initiating minor compaction (all files) 2024-11-09T02:24:15,333 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64fc150c6b20bf9e198fa51cd5e2ea2a/A in TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:15,333 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64fc150c6b20bf9e198fa51cd5e2ea2a/B in TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:15,333 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/b11790976f1f471bb3910198ca1d8673, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/af27a5e199314750a60083857c59c1fb, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/2c1d747b316f42e9a8bfd7249ad1375e] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp, totalSize=100.3 K 2024-11-09T02:24:15,333 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/bc7a41ca80324944aef0cbe1cd1319a7, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/5910788a59c946a5934b05e2d772eafc, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/8eca84cbedbe4c0d928797645205f47d] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp, totalSize=36.2 K 2024-11-09T02:24:15,333 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:15,333 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. files: [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/b11790976f1f471bb3910198ca1d8673, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/af27a5e199314750a60083857c59c1fb, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/2c1d747b316f42e9a8bfd7249ad1375e] 2024-11-09T02:24:15,333 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting bc7a41ca80324944aef0cbe1cd1319a7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731119048843 2024-11-09T02:24:15,333 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting b11790976f1f471bb3910198ca1d8673, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731119048843 2024-11-09T02:24:15,333 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 5910788a59c946a5934b05e2d772eafc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731119050918 2024-11-09T02:24:15,333 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting af27a5e199314750a60083857c59c1fb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731119050918 2024-11-09T02:24:15,334 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 8eca84cbedbe4c0d928797645205f47d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731119053033 2024-11-09T02:24:15,334 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c1d747b316f42e9a8bfd7249ad1375e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731119053033 2024-11-09T02:24:15,339 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:15,340 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64fc150c6b20bf9e198fa51cd5e2ea2a#B#compaction#477 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:24:15,340 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/55e105610ec3424a866f63a4d1bda693 is 50, key is test_row_0/B:col10/1731119053033/Put/seqid=0 2024-11-09T02:24:15,341 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110900a64e965f3f429aa54a8690b52a13e7_64fc150c6b20bf9e198fa51cd5e2ea2a store=[table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:15,347 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110900a64e965f3f429aa54a8690b52a13e7_64fc150c6b20bf9e198fa51cd5e2ea2a, store=[table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:15,347 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110900a64e965f3f429aa54a8690b52a13e7_64fc150c6b20bf9e198fa51cd5e2ea2a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:15,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742398_1574 (size=12915) 2024-11-09T02:24:15,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742399_1575 (size=4469) 2024-11-09T02:24:15,754 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64fc150c6b20bf9e198fa51cd5e2ea2a#A#compaction#478 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:24:15,755 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/db8f10daccdf4fee92a2cbe88a08e63b is 175, key is test_row_0/A:col10/1731119053033/Put/seqid=0 2024-11-09T02:24:15,759 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/55e105610ec3424a866f63a4d1bda693 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/55e105610ec3424a866f63a4d1bda693 2024-11-09T02:24:15,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742400_1576 (size=31869) 2024-11-09T02:24:15,764 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64fc150c6b20bf9e198fa51cd5e2ea2a/B of 64fc150c6b20bf9e198fa51cd5e2ea2a into 55e105610ec3424a866f63a4d1bda693(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:24:15,764 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:24:15,764 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., storeName=64fc150c6b20bf9e198fa51cd5e2ea2a/B, priority=13, startTime=1731119055332; duration=0sec 2024-11-09T02:24:15,764 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T02:24:15,764 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64fc150c6b20bf9e198fa51cd5e2ea2a:B 2024-11-09T02:24:15,765 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T02:24:15,765 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T02:24:15,765 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1541): 64fc150c6b20bf9e198fa51cd5e2ea2a/C is initiating minor compaction (all files) 2024-11-09T02:24:15,766 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64fc150c6b20bf9e198fa51cd5e2ea2a/C in TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:15,766 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/bd617ec810ea48c4b182687a4c229f74, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/3cfc47dd578a486289e93e96c5c01641, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/47f6709da0d340788d1bf462dd37e744] into tmpdir=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp, totalSize=36.2 K 2024-11-09T02:24:15,766 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting bd617ec810ea48c4b182687a4c229f74, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731119048843 2024-11-09T02:24:15,766 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cfc47dd578a486289e93e96c5c01641, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731119050918 2024-11-09T02:24:15,767 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] compactions.Compactor(225): Compacting 47f6709da0d340788d1bf462dd37e744, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731119053033 2024-11-09T02:24:15,773 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64fc150c6b20bf9e198fa51cd5e2ea2a#C#compaction#479 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T02:24:15,774 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/cc8a9d5f58bf410783f6d045fa42c2fd is 50, key is test_row_0/C:col10/1731119053033/Put/seqid=0 2024-11-09T02:24:15,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742401_1577 (size=12915) 2024-11-09T02:24:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46079 {}] regionserver.HRegion(8855): Flush requested on 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:15,832 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T02:24:15,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64fc150c6b20bf9e198fa51cd5e2ea2a 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-09T02:24:15,833 INFO [Thread-2256 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2256. 2024-11-09T02:24:15,833 DEBUG [Thread-2256 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:24:15,833 DEBUG [Thread-2256 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:15,833 DEBUG [Thread-2256 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:15,833 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:24:15,843 INFO [Thread-2260 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2260. 2024-11-09T02:24:15,843 INFO [Thread-2254 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2254. 2024-11-09T02:24:15,843 DEBUG [Thread-2254 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:24:15,843 DEBUG [Thread-2260 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:24:15,843 DEBUG [Thread-2254 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:15,843 DEBUG [Thread-2260 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:15,843 DEBUG [Thread-2254 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:15,843 DEBUG [Thread-2260 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:15,843 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:24:15,844 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:24:15,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411092e6f8f0c59a74d08a2f8fbe0f40ec1ad_64fc150c6b20bf9e198fa51cd5e2ea2a is 50, key is test_row_0/A:col10/1731119053683/Put/seqid=0 2024-11-09T02:24:15,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742402_1578 (size=12454) 2024-11-09T02:24:15,851 INFO [Thread-2258 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2258. 2024-11-09T02:24:15,851 DEBUG [Thread-2258 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:24:15,851 DEBUG [Thread-2258 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:15,851 DEBUG [Thread-2258 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:15,851 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:24:16,171 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/db8f10daccdf4fee92a2cbe88a08e63b as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/db8f10daccdf4fee92a2cbe88a08e63b 2024-11-09T02:24:16,178 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64fc150c6b20bf9e198fa51cd5e2ea2a/A of 64fc150c6b20bf9e198fa51cd5e2ea2a into db8f10daccdf4fee92a2cbe88a08e63b(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:24:16,178 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:24:16,178 INFO [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., storeName=64fc150c6b20bf9e198fa51cd5e2ea2a/A, priority=13, startTime=1731119055332; duration=0sec 2024-11-09T02:24:16,178 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:24:16,178 DEBUG [RS:0;53f286d3af3b:46079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64fc150c6b20bf9e198fa51cd5e2ea2a:A 2024-11-09T02:24:16,182 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/cc8a9d5f58bf410783f6d045fa42c2fd as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/cc8a9d5f58bf410783f6d045fa42c2fd 2024-11-09T02:24:16,189 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64fc150c6b20bf9e198fa51cd5e2ea2a/C of 64fc150c6b20bf9e198fa51cd5e2ea2a into cc8a9d5f58bf410783f6d045fa42c2fd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T02:24:16,189 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:24:16,189 INFO [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., storeName=64fc150c6b20bf9e198fa51cd5e2ea2a/C, priority=13, startTime=1731119055332; duration=0sec 2024-11-09T02:24:16,189 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T02:24:16,189 DEBUG [RS:0;53f286d3af3b:46079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64fc150c6b20bf9e198fa51cd5e2ea2a:C 2024-11-09T02:24:16,249 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:24:16,255 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411092e6f8f0c59a74d08a2f8fbe0f40ec1ad_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092e6f8f0c59a74d08a2f8fbe0f40ec1ad_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:16,257 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/4281a74257e444738ffbaca804b91761, store: [table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:16,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/4281a74257e444738ffbaca804b91761 is 175, key is test_row_0/A:col10/1731119053683/Put/seqid=0 2024-11-09T02:24:16,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742403_1579 (size=31255) 2024-11-09T02:24:16,663 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/4281a74257e444738ffbaca804b91761 2024-11-09T02:24:16,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/8c5333c023f74cd8a61399d580f8b4c0 is 50, key is test_row_0/B:col10/1731119053683/Put/seqid=0 2024-11-09T02:24:16,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742404_1580 (size=12301) 2024-11-09T02:24:17,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/8c5333c023f74cd8a61399d580f8b4c0 2024-11-09T02:24:17,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/019fa74757de422188a1e921b5a5cc07 is 50, key is test_row_0/C:col10/1731119053683/Put/seqid=0 2024-11-09T02:24:17,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742405_1581 (size=12301) 2024-11-09T02:24:17,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/019fa74757de422188a1e921b5a5cc07 2024-11-09T02:24:17,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/4281a74257e444738ffbaca804b91761 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/4281a74257e444738ffbaca804b91761 2024-11-09T02:24:17,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/4281a74257e444738ffbaca804b91761, entries=150, sequenceid=291, filesize=30.5 K 2024-11-09T02:24:17,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/8c5333c023f74cd8a61399d580f8b4c0 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/8c5333c023f74cd8a61399d580f8b4c0 2024-11-09T02:24:17,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/8c5333c023f74cd8a61399d580f8b4c0, entries=150, sequenceid=291, filesize=12.0 K 2024-11-09T02:24:17,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/019fa74757de422188a1e921b5a5cc07 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/019fa74757de422188a1e921b5a5cc07 2024-11-09T02:24:17,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/019fa74757de422188a1e921b5a5cc07, entries=150, sequenceid=291, filesize=12.0 K 2024-11-09T02:24:17,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=20.13 KB/20610 for 64fc150c6b20bf9e198fa51cd5e2ea2a in 1692ms, sequenceid=291, compaction requested=false 2024-11-09T02:24:17,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: 2024-11-09T02:24:19,135 INFO [Thread-2262 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2262. 2024-11-09T02:24:19,135 DEBUG [Thread-2262 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T02:24:19,135 DEBUG [Thread-2262 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:19,136 DEBUG [Thread-2262 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:19,136 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:24:19,136 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T02:24:19,136 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-09T02:24:19,136 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-09T02:24:19,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-09T02:24:19,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-09T02:24:19,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-09T02:24:19,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T02:24:19,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4005 2024-11-09T02:24:19,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3914 2024-11-09T02:24:19,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3864 2024-11-09T02:24:19,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4019 2024-11-09T02:24:19,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3910 2024-11-09T02:24:19,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T02:24:19,137 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T02:24:19,137 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:24:19,138 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:19,138 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:19,138 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:24:19,140 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T02:24:19,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T02:24:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-09T02:24:19,146 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731119059145"}]},"ts":"1731119059145"} 2024-11-09T02:24:19,148 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T02:24:19,148 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T02:24:19,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T02:24:19,151 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=64fc150c6b20bf9e198fa51cd5e2ea2a, UNASSIGN}] 2024-11-09T02:24:19,152 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=64fc150c6b20bf9e198fa51cd5e2ea2a, UNASSIGN 2024-11-09T02:24:19,153 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=64fc150c6b20bf9e198fa51cd5e2ea2a, regionState=CLOSING, regionLocation=53f286d3af3b,46079,1731118875223 2024-11-09T02:24:19,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=64fc150c6b20bf9e198fa51cd5e2ea2a, UNASSIGN because future has completed 2024-11-09T02:24:19,155 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T02:24:19,155 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure 64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223}] 2024-11-09T02:24:19,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-09T02:24:19,309 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(122): Close 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:19,309 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T02:24:19,309 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1722): Closing 64fc150c6b20bf9e198fa51cd5e2ea2a, disabling compactions & flushes 2024-11-09T02:24:19,309 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:19,309 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:19,309 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. after waiting 0 ms 2024-11-09T02:24:19,309 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:19,309 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(2902): Flushing 64fc150c6b20bf9e198fa51cd5e2ea2a 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-09T02:24:19,319 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411092df80f92fed840bc954c62bee30f00b0_64fc150c6b20bf9e198fa51cd5e2ea2a is 50, key is test_row_1/A:col10/1731119055850/Put/seqid=0 2024-11-09T02:24:19,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742406_1582 (size=9914) 2024-11-09T02:24:19,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-09T02:24:19,724 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:24:19,731 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411092df80f92fed840bc954c62bee30f00b0_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092df80f92fed840bc954c62bee30f00b0_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:19,733 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/ebf8632fddb347a38ba56f149f3aa607, store: [table=TestAcidGuarantees family=A region=64fc150c6b20bf9e198fa51cd5e2ea2a] 2024-11-09T02:24:19,734 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/ebf8632fddb347a38ba56f149f3aa607 is 175, key is test_row_1/A:col10/1731119055850/Put/seqid=0 2024-11-09T02:24:19,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742407_1583 (size=22561) 2024-11-09T02:24:19,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-09T02:24:20,141 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=300, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/ebf8632fddb347a38ba56f149f3aa607 2024-11-09T02:24:20,196 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/2923b1fcf60e4e9491df9770998ed05b is 50, key is test_row_1/B:col10/1731119055850/Put/seqid=0 2024-11-09T02:24:20,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742408_1584 (size=9857) 2024-11-09T02:24:20,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-09T02:24:20,602 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/2923b1fcf60e4e9491df9770998ed05b 2024-11-09T02:24:20,615 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/b147ebdc1e244d6ba28ab7819f41aa44 is 50, key is test_row_1/C:col10/1731119055850/Put/seqid=0 2024-11-09T02:24:20,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742409_1585 (size=9857) 2024-11-09T02:24:21,021 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/b147ebdc1e244d6ba28ab7819f41aa44 2024-11-09T02:24:21,031 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/A/ebf8632fddb347a38ba56f149f3aa607 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/ebf8632fddb347a38ba56f149f3aa607 2024-11-09T02:24:21,036 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/ebf8632fddb347a38ba56f149f3aa607, entries=100, sequenceid=300, filesize=22.0 K 2024-11-09T02:24:21,037 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/B/2923b1fcf60e4e9491df9770998ed05b as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/2923b1fcf60e4e9491df9770998ed05b 2024-11-09T02:24:21,041 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/2923b1fcf60e4e9491df9770998ed05b, entries=100, sequenceid=300, filesize=9.6 K 2024-11-09T02:24:21,042 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/.tmp/C/b147ebdc1e244d6ba28ab7819f41aa44 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/b147ebdc1e244d6ba28ab7819f41aa44 2024-11-09T02:24:21,044 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/b147ebdc1e244d6ba28ab7819f41aa44, entries=100, sequenceid=300, filesize=9.6 K 2024-11-09T02:24:21,045 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 64fc150c6b20bf9e198fa51cd5e2ea2a in 1736ms, sequenceid=300, compaction requested=true 2024-11-09T02:24:21,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/7d5c9fb774db4fabb48be03251cd9247, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/cd5539ae0560446f9e39dfd9ea4293c9, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/0c9f8d212eb240f284991a073823d67c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/ff3ba5a95af843df8df552adf7bc458c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/cbffb8aece114e9c9a07e2fbb78c76a0, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/314b85e7e50e48f6a99e9185efdff2a2, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/7b9e245db2154eeebf307bf3daf0c831, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/b11b49ea6ca642e1ba0970c3d759a30d, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/4ef5b9b8380a4629a6a57d52bb0052ab, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/c7d673be91194bfc897533bf154f8292, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/a8c095d3bfa34139b73be89b7cb23ead, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/d95ce1e76195434e9cadc281c950cf93, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/eed9a67bd18948569cbd67f91fe403b6, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/8bbf3108fc2f43cf9b19a91af9536d11, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/aac24c7335704b85b7d4cb19f5bdcd26, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/b11790976f1f471bb3910198ca1d8673, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/af27a5e199314750a60083857c59c1fb, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/2c1d747b316f42e9a8bfd7249ad1375e] to archive 2024-11-09T02:24:21,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T02:24:21,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/7d5c9fb774db4fabb48be03251cd9247 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/7d5c9fb774db4fabb48be03251cd9247 2024-11-09T02:24:21,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/cd5539ae0560446f9e39dfd9ea4293c9 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/cd5539ae0560446f9e39dfd9ea4293c9 2024-11-09T02:24:21,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/0c9f8d212eb240f284991a073823d67c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/0c9f8d212eb240f284991a073823d67c 2024-11-09T02:24:21,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/ff3ba5a95af843df8df552adf7bc458c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/ff3ba5a95af843df8df552adf7bc458c 2024-11-09T02:24:21,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/cbffb8aece114e9c9a07e2fbb78c76a0 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/cbffb8aece114e9c9a07e2fbb78c76a0 2024-11-09T02:24:21,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/314b85e7e50e48f6a99e9185efdff2a2 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/314b85e7e50e48f6a99e9185efdff2a2 2024-11-09T02:24:21,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/7b9e245db2154eeebf307bf3daf0c831 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/7b9e245db2154eeebf307bf3daf0c831 2024-11-09T02:24:21,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/b11b49ea6ca642e1ba0970c3d759a30d to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/b11b49ea6ca642e1ba0970c3d759a30d 2024-11-09T02:24:21,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/4ef5b9b8380a4629a6a57d52bb0052ab to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/4ef5b9b8380a4629a6a57d52bb0052ab 2024-11-09T02:24:21,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/c7d673be91194bfc897533bf154f8292 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/c7d673be91194bfc897533bf154f8292 2024-11-09T02:24:21,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/a8c095d3bfa34139b73be89b7cb23ead to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/a8c095d3bfa34139b73be89b7cb23ead 2024-11-09T02:24:21,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/d95ce1e76195434e9cadc281c950cf93 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/d95ce1e76195434e9cadc281c950cf93 2024-11-09T02:24:21,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/eed9a67bd18948569cbd67f91fe403b6 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/eed9a67bd18948569cbd67f91fe403b6 2024-11-09T02:24:21,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/8bbf3108fc2f43cf9b19a91af9536d11 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/8bbf3108fc2f43cf9b19a91af9536d11 2024-11-09T02:24:21,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/aac24c7335704b85b7d4cb19f5bdcd26 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/aac24c7335704b85b7d4cb19f5bdcd26 2024-11-09T02:24:21,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/b11790976f1f471bb3910198ca1d8673 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/b11790976f1f471bb3910198ca1d8673 2024-11-09T02:24:21,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/af27a5e199314750a60083857c59c1fb to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/af27a5e199314750a60083857c59c1fb 2024-11-09T02:24:21,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/2c1d747b316f42e9a8bfd7249ad1375e to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/2c1d747b316f42e9a8bfd7249ad1375e 2024-11-09T02:24:21,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/56207b358da44633981e980148f73f21, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/26777c5fb00749bda90b15c59e2243fb, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/8a43b68dedb44765bf182779a1b2f638, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/4f9edf3edcdd41b9861b7e4c878c52e0, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/cb76a7f13a8b4bcfbe8fa8acbbb163e0, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/f16a1bfa3ddb4d6da24a288f526316a3, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/91d6cb4b545d4d9a8645fd491b0784f4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/960cf28778af4efbb200a72e201c9a07, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/9be10c39973e440dbc770a47a4210560, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/383d21ee398346ce86506541dbf6193e, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/b6194983120145ff82809595d41b5570, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/b0fcfe8cb6414c72ad6c243686d0ee0e, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/5dada534fae243a58c1b2d59861f5ed4, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/ab5f122e499c4fee9148c71705b0256a, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/bc7a41ca80324944aef0cbe1cd1319a7, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/0fbe81ca519c4e50853f06068d674863, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/5910788a59c946a5934b05e2d772eafc, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/8eca84cbedbe4c0d928797645205f47d] to archive 2024-11-09T02:24:21,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T02:24:21,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/56207b358da44633981e980148f73f21 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/56207b358da44633981e980148f73f21 2024-11-09T02:24:21,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/26777c5fb00749bda90b15c59e2243fb to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/26777c5fb00749bda90b15c59e2243fb 2024-11-09T02:24:21,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/8a43b68dedb44765bf182779a1b2f638 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/8a43b68dedb44765bf182779a1b2f638 2024-11-09T02:24:21,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/4f9edf3edcdd41b9861b7e4c878c52e0 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/4f9edf3edcdd41b9861b7e4c878c52e0 2024-11-09T02:24:21,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/cb76a7f13a8b4bcfbe8fa8acbbb163e0 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/cb76a7f13a8b4bcfbe8fa8acbbb163e0 2024-11-09T02:24:21,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/f16a1bfa3ddb4d6da24a288f526316a3 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/f16a1bfa3ddb4d6da24a288f526316a3 2024-11-09T02:24:21,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/91d6cb4b545d4d9a8645fd491b0784f4 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/91d6cb4b545d4d9a8645fd491b0784f4 2024-11-09T02:24:21,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/960cf28778af4efbb200a72e201c9a07 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/960cf28778af4efbb200a72e201c9a07 2024-11-09T02:24:21,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/9be10c39973e440dbc770a47a4210560 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/9be10c39973e440dbc770a47a4210560 2024-11-09T02:24:21,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/383d21ee398346ce86506541dbf6193e to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/383d21ee398346ce86506541dbf6193e 2024-11-09T02:24:21,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/b6194983120145ff82809595d41b5570 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/b6194983120145ff82809595d41b5570 2024-11-09T02:24:21,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/b0fcfe8cb6414c72ad6c243686d0ee0e to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/b0fcfe8cb6414c72ad6c243686d0ee0e 2024-11-09T02:24:21,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/5dada534fae243a58c1b2d59861f5ed4 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/5dada534fae243a58c1b2d59861f5ed4 2024-11-09T02:24:21,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/ab5f122e499c4fee9148c71705b0256a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/ab5f122e499c4fee9148c71705b0256a 2024-11-09T02:24:21,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/bc7a41ca80324944aef0cbe1cd1319a7 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/bc7a41ca80324944aef0cbe1cd1319a7 2024-11-09T02:24:21,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/0fbe81ca519c4e50853f06068d674863 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/0fbe81ca519c4e50853f06068d674863 2024-11-09T02:24:21,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/5910788a59c946a5934b05e2d772eafc to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/5910788a59c946a5934b05e2d772eafc 2024-11-09T02:24:21,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/8eca84cbedbe4c0d928797645205f47d to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/8eca84cbedbe4c0d928797645205f47d 2024-11-09T02:24:21,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/ae6ed4017c324bafaacad2d4b667ad69, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/f988146a9870442a84efeeda3169ad7c, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/45a9c6f06e234ab18d8539a52706a3e9, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/17c8bb6e7c5b42c1a2c417717c2c44b0, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/5407eba030344e30be69f3545f174bf2, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/23ee1205664c44598843cdc8ef74e217, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/353906c611e4482983c4fffc8d865e0b, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/867b7bf2a3494533823d593e1e151cb2, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/1c7180544d2b4e8887387b0548bd4590, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/1edfafd88f29485ca50977c31e81ad15, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/76cc6c04ada74a8abcc6929fb69ee507, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/2b13ffac795a4992849a8654d936db67, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/5c92c0e862fc45ec913ffc2ec1691362, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/bb202127b82b44cb84d6b53caba11b98, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/bd617ec810ea48c4b182687a4c229f74, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/b795f648fd5a4430ae419b9218fd6066, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/3cfc47dd578a486289e93e96c5c01641, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/47f6709da0d340788d1bf462dd37e744] to archive 2024-11-09T02:24:21,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T02:24:21,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/ae6ed4017c324bafaacad2d4b667ad69 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/ae6ed4017c324bafaacad2d4b667ad69 2024-11-09T02:24:21,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/f988146a9870442a84efeeda3169ad7c to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/f988146a9870442a84efeeda3169ad7c 2024-11-09T02:24:21,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/45a9c6f06e234ab18d8539a52706a3e9 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/45a9c6f06e234ab18d8539a52706a3e9 2024-11-09T02:24:21,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/17c8bb6e7c5b42c1a2c417717c2c44b0 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/17c8bb6e7c5b42c1a2c417717c2c44b0 2024-11-09T02:24:21,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/5407eba030344e30be69f3545f174bf2 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/5407eba030344e30be69f3545f174bf2 2024-11-09T02:24:21,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/23ee1205664c44598843cdc8ef74e217 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/23ee1205664c44598843cdc8ef74e217 2024-11-09T02:24:21,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/353906c611e4482983c4fffc8d865e0b to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/353906c611e4482983c4fffc8d865e0b 2024-11-09T02:24:21,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/867b7bf2a3494533823d593e1e151cb2 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/867b7bf2a3494533823d593e1e151cb2 2024-11-09T02:24:21,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/1c7180544d2b4e8887387b0548bd4590 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/1c7180544d2b4e8887387b0548bd4590 2024-11-09T02:24:21,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/1edfafd88f29485ca50977c31e81ad15 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/1edfafd88f29485ca50977c31e81ad15 2024-11-09T02:24:21,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/76cc6c04ada74a8abcc6929fb69ee507 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/76cc6c04ada74a8abcc6929fb69ee507 2024-11-09T02:24:21,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/2b13ffac795a4992849a8654d936db67 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/2b13ffac795a4992849a8654d936db67 2024-11-09T02:24:21,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/5c92c0e862fc45ec913ffc2ec1691362 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/5c92c0e862fc45ec913ffc2ec1691362 2024-11-09T02:24:21,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/bb202127b82b44cb84d6b53caba11b98 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/bb202127b82b44cb84d6b53caba11b98 2024-11-09T02:24:21,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/bd617ec810ea48c4b182687a4c229f74 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/bd617ec810ea48c4b182687a4c229f74 2024-11-09T02:24:21,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/b795f648fd5a4430ae419b9218fd6066 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/b795f648fd5a4430ae419b9218fd6066 2024-11-09T02:24:21,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/3cfc47dd578a486289e93e96c5c01641 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/3cfc47dd578a486289e93e96c5c01641 2024-11-09T02:24:21,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/47f6709da0d340788d1bf462dd37e744 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/47f6709da0d340788d1bf462dd37e744 2024-11-09T02:24:21,108 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/recovered.edits/303.seqid, newMaxSeqId=303, maxSeqId=4 2024-11-09T02:24:21,109 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. 2024-11-09T02:24:21,109 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1676): Region close journal for 64fc150c6b20bf9e198fa51cd5e2ea2a: Waiting for close lock at 1731119059309Running coprocessor pre-close hooks at 1731119059309Disabling compacts and flushes for region at 1731119059309Disabling writes for close at 1731119059309Obtaining lock to block concurrent updates at 1731119059309Preparing flush snapshotting stores in 64fc150c6b20bf9e198fa51cd5e2ea2a at 1731119059309Finished memstore snapshotting TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731119059310 (+1 ms)Flushing stores of TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a. at 1731119059311 (+1 ms)Flushing 64fc150c6b20bf9e198fa51cd5e2ea2a/A: creating writer at 1731119059311Flushing mob file 64fc150c6b20bf9e198fa51cd5e2ea2a/A: appending metadata at 1731119059319 (+8 ms)Flushing mob file 64fc150c6b20bf9e198fa51cd5e2ea2a/A: closing flushed file at 1731119059319Flushing 64fc150c6b20bf9e198fa51cd5e2ea2a/A: appending metadata at 1731119059733 (+414 ms)Flushing 64fc150c6b20bf9e198fa51cd5e2ea2a/A: closing flushed file at 1731119059733Flushing 64fc150c6b20bf9e198fa51cd5e2ea2a/B: creating writer at 1731119060191 (+458 ms)Flushing 64fc150c6b20bf9e198fa51cd5e2ea2a/B: appending metadata at 1731119060196 (+5 ms)Flushing 64fc150c6b20bf9e198fa51cd5e2ea2a/B: closing flushed file at 1731119060196Flushing 64fc150c6b20bf9e198fa51cd5e2ea2a/C: creating writer at 1731119060610 (+414 ms)Flushing 64fc150c6b20bf9e198fa51cd5e2ea2a/C: appending metadata at 1731119060614 (+4 ms)Flushing 64fc150c6b20bf9e198fa51cd5e2ea2a/C: closing flushed file at 1731119060614Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21d18841: reopening flushed file at 1731119061030 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5282bd8a: reopening flushed file at 1731119061036 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76e450ed: reopening flushed file at 1731119061041 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 64fc150c6b20bf9e198fa51cd5e2ea2a in 1736ms, sequenceid=300, compaction requested=true at 1731119061045 (+4 ms)Writing region close event to WAL at 1731119061106 (+61 ms)Running coprocessor post-close hooks at 1731119061108 (+2 ms)Closed at 1731119061108 2024-11-09T02:24:21,110 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(157): Closed 64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:21,111 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=64fc150c6b20bf9e198fa51cd5e2ea2a, regionState=CLOSED 2024-11-09T02:24:21,112 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure 64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 because future has completed 2024-11-09T02:24:21,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-09T02:24:21,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseRegionProcedure 64fc150c6b20bf9e198fa51cd5e2ea2a, server=53f286d3af3b,46079,1731118875223 in 1.9580 sec 2024-11-09T02:24:21,116 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=162, resume processing ppid=161 2024-11-09T02:24:21,116 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, ppid=161, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=64fc150c6b20bf9e198fa51cd5e2ea2a, UNASSIGN in 1.9630 sec 2024-11-09T02:24:21,118 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-09T02:24:21,118 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9670 sec 2024-11-09T02:24:21,119 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731119061119"}]},"ts":"1731119061119"} 2024-11-09T02:24:21,121 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T02:24:21,121 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T02:24:21,123 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9810 sec 2024-11-09T02:24:21,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-09T02:24:21,292 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:24:21,293 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T02:24:21,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:24:21,297 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:24:21,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-09T02:24:21,298 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=164, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:24:21,302 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:21,305 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A, FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B, FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C, FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/recovered.edits] 2024-11-09T02:24:21,308 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/4281a74257e444738ffbaca804b91761 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/4281a74257e444738ffbaca804b91761 2024-11-09T02:24:21,310 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/db8f10daccdf4fee92a2cbe88a08e63b to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/db8f10daccdf4fee92a2cbe88a08e63b 2024-11-09T02:24:21,311 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/ebf8632fddb347a38ba56f149f3aa607 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/A/ebf8632fddb347a38ba56f149f3aa607 2024-11-09T02:24:21,313 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/2923b1fcf60e4e9491df9770998ed05b to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/2923b1fcf60e4e9491df9770998ed05b 2024-11-09T02:24:21,314 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/55e105610ec3424a866f63a4d1bda693 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/55e105610ec3424a866f63a4d1bda693 2024-11-09T02:24:21,315 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/8c5333c023f74cd8a61399d580f8b4c0 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/B/8c5333c023f74cd8a61399d580f8b4c0 2024-11-09T02:24:21,317 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/019fa74757de422188a1e921b5a5cc07 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/019fa74757de422188a1e921b5a5cc07 2024-11-09T02:24:21,318 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/b147ebdc1e244d6ba28ab7819f41aa44 to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/b147ebdc1e244d6ba28ab7819f41aa44 2024-11-09T02:24:21,318 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/cc8a9d5f58bf410783f6d045fa42c2fd to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/C/cc8a9d5f58bf410783f6d045fa42c2fd 2024-11-09T02:24:21,320 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/recovered.edits/303.seqid to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a/recovered.edits/303.seqid 2024-11-09T02:24:21,321 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/default/TestAcidGuarantees/64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:21,321 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T02:24:21,321 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T02:24:21,322 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-09T02:24:21,324 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110905238632915a41e4be25294df4d21b8d_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110905238632915a41e4be25294df4d21b8d_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:21,325 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091f873a3d71a348f7a09536fc98bed344_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091f873a3d71a348f7a09536fc98bed344_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:21,326 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109282c7ffc7b084130995081ec43333396_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109282c7ffc7b084130995081ec43333396_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:21,327 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092df80f92fed840bc954c62bee30f00b0_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092df80f92fed840bc954c62bee30f00b0_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:21,328 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092e6f8f0c59a74d08a2f8fbe0f40ec1ad_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092e6f8f0c59a74d08a2f8fbe0f40ec1ad_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:21,329 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109352575967ad54b01a410c406d4d2021a_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109352575967ad54b01a410c406d4d2021a_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:21,330 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110936ed756c4dc54886b8602585815b97ee_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110936ed756c4dc54886b8602585815b97ee_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:21,331 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109538c4f9612ae4bfcbaafb7142a57a939_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109538c4f9612ae4bfcbaafb7142a57a939_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:21,332 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095f4869eca00140349bae8b47f18600b7_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095f4869eca00140349bae8b47f18600b7_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:21,333 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110973169e5b5b554a2c8c3ae6eaec7d245d_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110973169e5b5b554a2c8c3ae6eaec7d245d_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:21,333 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097e02b375a09741bb9690b42c56461237_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097e02b375a09741bb9690b42c56461237_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:21,334 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097fca22ea34714f2e8b35bc27fb7b3a77_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097fca22ea34714f2e8b35bc27fb7b3a77_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:21,335 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110985363a86ae884eb8b2d19ea444f9984d_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110985363a86ae884eb8b2d19ea444f9984d_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:21,336 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109900e0d5957f34f18a23499ec76cd1de6_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109900e0d5957f34f18a23499ec76cd1de6_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:21,336 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099deb5f3deb204098a2fcf796a55698dc_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099deb5f3deb204098a2fcf796a55698dc_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:21,337 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ebadd40418214691ab0638f2780bc196_64fc150c6b20bf9e198fa51cd5e2ea2a to hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ebadd40418214691ab0638f2780bc196_64fc150c6b20bf9e198fa51cd5e2ea2a 2024-11-09T02:24:21,338 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T02:24:21,339 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=164, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:24:21,341 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T02:24:21,343 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T02:24:21,344 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=164, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:24:21,344 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T02:24:21,344 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731119061344"}]},"ts":"9223372036854775807"} 2024-11-09T02:24:21,346 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T02:24:21,346 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 64fc150c6b20bf9e198fa51cd5e2ea2a, NAME => 'TestAcidGuarantees,,1731119031428.64fc150c6b20bf9e198fa51cd5e2ea2a.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T02:24:21,346 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T02:24:21,346 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731119061346"}]},"ts":"9223372036854775807"} 2024-11-09T02:24:21,348 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T02:24:21,348 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=164, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T02:24:21,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 55 msec 2024-11-09T02:24:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45041 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-09T02:24:21,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T02:24:21,402 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T02:24:21,414 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=238 (was 238), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=367 (was 403), ProcessCount=11 (was 11), AvailableMemoryMB=2864 (was 2880) 2024-11-09T02:24:21,415 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-09T02:24:21,415 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T02:24:21,415 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:24:21,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:21,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:21,415 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:24:21,415 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-09T02:24:21,415 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1336910459, stopped=false 2024-11-09T02:24:21,416 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=53f286d3af3b,45041,1731118874457 2024-11-09T02:24:21,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T02:24:21,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46079-0x1011d65b7800001, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T02:24:21,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:24:21,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46079-0x1011d65b7800001, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:24:21,477 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:24:21,479 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T02:24:21,479 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T02:24:21,479 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:24:21,479 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46079-0x1011d65b7800001, quorum=127.0.0.1:62339, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T02:24:21,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:21,480 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '53f286d3af3b,46079,1731118875223' ***** 2024-11-09T02:24:21,480 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-09T02:24:21,481 INFO [RS:0;53f286d3af3b:46079 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-09T02:24:21,481 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-09T02:24:21,481 INFO [RS:0;53f286d3af3b:46079 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-09T02:24:21,481 INFO [RS:0;53f286d3af3b:46079 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-09T02:24:21,481 INFO [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer(959): stopping server 53f286d3af3b,46079,1731118875223 2024-11-09T02:24:21,482 INFO [RS:0;53f286d3af3b:46079 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T02:24:21,482 INFO [RS:0;53f286d3af3b:46079 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;53f286d3af3b:46079. 2024-11-09T02:24:21,482 DEBUG [RS:0;53f286d3af3b:46079 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:24:21,482 DEBUG [RS:0;53f286d3af3b:46079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:21,482 INFO [RS:0;53f286d3af3b:46079 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-09T02:24:21,482 INFO [RS:0;53f286d3af3b:46079 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-09T02:24:21,482 INFO [RS:0;53f286d3af3b:46079 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-09T02:24:21,482 INFO [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-09T02:24:21,483 INFO [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-09T02:24:21,483 DEBUG [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-09T02:24:21,483 DEBUG [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T02:24:21,483 DEBUG [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T02:24:21,483 INFO [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T02:24:21,483 DEBUG [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T02:24:21,484 DEBUG [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T02:24:21,484 DEBUG [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T02:24:21,484 INFO [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-09T02:24:21,502 DEBUG [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/1588230740/.tmp/info/6aabf735495e449fa923211a0c934f41 is 95, key is TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe./info:/1731118908187/DeleteFamily/seqid=0 2024-11-09T02:24:21,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742410_1586 (size=6514) 2024-11-09T02:24:21,546 INFO [regionserver/53f286d3af3b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T02:24:21,684 DEBUG [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T02:24:21,884 DEBUG [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T02:24:21,907 INFO [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/1588230740/.tmp/info/6aabf735495e449fa923211a0c934f41 2024-11-09T02:24:21,933 DEBUG [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/1588230740/.tmp/ns/e90d2eef2cd34752960232e7b5d2bf3d is 93, key is TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe./ns:/1731118908145/DeleteFamily/seqid=0 2024-11-09T02:24:21,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742411_1587 (size=5906) 2024-11-09T02:24:22,084 DEBUG [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T02:24:22,285 DEBUG [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T02:24:22,337 INFO [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/1588230740/.tmp/ns/e90d2eef2cd34752960232e7b5d2bf3d 2024-11-09T02:24:22,366 DEBUG [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/1588230740/.tmp/rep_barrier/1b81570714a543d991f0c534d0f147f6 is 102, key is TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe./rep_barrier:/1731118908145/DeleteFamily/seqid=0 2024-11-09T02:24:22,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742412_1588 (size=6025) 2024-11-09T02:24:22,485 DEBUG [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T02:24:22,490 INFO [regionserver/53f286d3af3b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-09T02:24:22,490 INFO [regionserver/53f286d3af3b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-09T02:24:22,685 DEBUG [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T02:24:22,771 INFO [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/1588230740/.tmp/rep_barrier/1b81570714a543d991f0c534d0f147f6 2024-11-09T02:24:22,798 DEBUG [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/1588230740/.tmp/table/f5af382a7b024677a9303d98fb48e64a is 96, key is TestAcidGuarantees,,1731118879603.aad4fc6835c8f9bd4339c4a54a2c56fe./table:/1731118908145/DeleteFamily/seqid=0 2024-11-09T02:24:22,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742413_1589 (size=5936) 2024-11-09T02:24:22,885 DEBUG [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T02:24:23,086 DEBUG [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T02:24:23,202 INFO [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/1588230740/.tmp/table/f5af382a7b024677a9303d98fb48e64a 2024-11-09T02:24:23,213 DEBUG [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/1588230740/.tmp/info/6aabf735495e449fa923211a0c934f41 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/1588230740/info/6aabf735495e449fa923211a0c934f41 2024-11-09T02:24:23,216 INFO [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/1588230740/info/6aabf735495e449fa923211a0c934f41, entries=12, sequenceid=90, filesize=6.4 K 2024-11-09T02:24:23,217 DEBUG [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/1588230740/.tmp/ns/e90d2eef2cd34752960232e7b5d2bf3d as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/1588230740/ns/e90d2eef2cd34752960232e7b5d2bf3d 2024-11-09T02:24:23,221 INFO [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/1588230740/ns/e90d2eef2cd34752960232e7b5d2bf3d, entries=8, sequenceid=90, filesize=5.8 K 2024-11-09T02:24:23,222 DEBUG [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/1588230740/.tmp/rep_barrier/1b81570714a543d991f0c534d0f147f6 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/1588230740/rep_barrier/1b81570714a543d991f0c534d0f147f6 2024-11-09T02:24:23,225 INFO [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/1588230740/rep_barrier/1b81570714a543d991f0c534d0f147f6, entries=6, sequenceid=90, filesize=5.9 K 2024-11-09T02:24:23,226 DEBUG [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/1588230740/.tmp/table/f5af382a7b024677a9303d98fb48e64a as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/1588230740/table/f5af382a7b024677a9303d98fb48e64a 2024-11-09T02:24:23,230 INFO [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/1588230740/table/f5af382a7b024677a9303d98fb48e64a, entries=7, sequenceid=90, filesize=5.8 K 2024-11-09T02:24:23,231 INFO [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1747ms, sequenceid=90, compaction requested=false 2024-11-09T02:24:23,236 DEBUG [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-09T02:24:23,237 DEBUG [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T02:24:23,237 INFO [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T02:24:23,237 DEBUG [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731119061483Running coprocessor pre-close hooks at 1731119061483Disabling compacts and flushes for region at 1731119061483Disabling writes for close at 1731119061484 (+1 ms)Obtaining lock to block concurrent updates at 1731119061484Preparing flush snapshotting stores in 1588230740 at 1731119061484Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731119061484Flushing stores of hbase:meta,,1.1588230740 at 1731119061484Flushing 1588230740/info: creating writer at 1731119061485 (+1 ms)Flushing 1588230740/info: appending metadata at 1731119061502 (+17 ms)Flushing 1588230740/info: closing flushed file at 1731119061502Flushing 1588230740/ns: creating writer at 1731119061917 (+415 ms)Flushing 1588230740/ns: appending metadata at 1731119061932 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731119061932Flushing 1588230740/rep_barrier: creating writer at 1731119062347 (+415 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731119062366 (+19 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731119062366Flushing 1588230740/table: creating writer at 1731119062780 (+414 ms)Flushing 1588230740/table: appending metadata at 1731119062797 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731119062797Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e4fa48f: reopening flushed file at 1731119063212 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7253561c: reopening flushed file at 1731119063217 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@733f5202: reopening flushed file at 1731119063221 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7595acc4: reopening flushed file at 1731119063226 (+5 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1747ms, sequenceid=90, compaction requested=false at 1731119063231 (+5 ms)Writing region close event to WAL at 1731119063233 (+2 ms)Running coprocessor post-close hooks at 1731119063236 (+3 ms)Closed at 1731119063237 (+1 ms) 2024-11-09T02:24:23,237 DEBUG [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-09T02:24:23,286 INFO [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer(976): stopping server 53f286d3af3b,46079,1731118875223; all regions closed. 2024-11-09T02:24:23,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741834_1010 (size=25344) 2024-11-09T02:24:23,299 DEBUG [RS:0;53f286d3af3b:46079 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/oldWALs 2024-11-09T02:24:23,300 INFO [RS:0;53f286d3af3b:46079 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 53f286d3af3b%2C46079%2C1731118875223.meta:.meta(num 1731118879135) 2024-11-09T02:24:23,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741833_1009 (size=13134918) 2024-11-09T02:24:23,305 DEBUG [RS:0;53f286d3af3b:46079 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/oldWALs 2024-11-09T02:24:23,305 INFO [RS:0;53f286d3af3b:46079 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 53f286d3af3b%2C46079%2C1731118875223:(num 1731118878587) 2024-11-09T02:24:23,305 DEBUG [RS:0;53f286d3af3b:46079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:24:23,305 INFO [RS:0;53f286d3af3b:46079 {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T02:24:23,305 INFO [RS:0;53f286d3af3b:46079 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T02:24:23,305 INFO [RS:0;53f286d3af3b:46079 {}] hbase.ChoreService(370): Chore service for: regionserver/53f286d3af3b:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-09T02:24:23,306 INFO [RS:0;53f286d3af3b:46079 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T02:24:23,306 INFO [regionserver/53f286d3af3b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T02:24:23,306 INFO [RS:0;53f286d3af3b:46079 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46079 2024-11-09T02:24:23,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46079-0x1011d65b7800001, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/53f286d3af3b,46079,1731118875223 2024-11-09T02:24:23,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T02:24:23,372 INFO [RS:0;53f286d3af3b:46079 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T02:24:23,382 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [53f286d3af3b,46079,1731118875223] 2024-11-09T02:24:23,392 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/53f286d3af3b,46079,1731118875223 already deleted, retry=false 2024-11-09T02:24:23,392 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 53f286d3af3b,46079,1731118875223 expired; onlineServers=0 2024-11-09T02:24:23,392 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '53f286d3af3b,45041,1731118874457' ***** 2024-11-09T02:24:23,392 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-09T02:24:23,392 INFO [M:0;53f286d3af3b:45041 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T02:24:23,393 INFO [M:0;53f286d3af3b:45041 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T02:24:23,393 DEBUG [M:0;53f286d3af3b:45041 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-09T02:24:23,393 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-09T02:24:23,393 DEBUG [M:0;53f286d3af3b:45041 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-09T02:24:23,393 DEBUG [master/53f286d3af3b:0:becomeActiveMaster-HFileCleaner.small.0-1731118878227 {}] cleaner.HFileCleaner(306): Exit Thread[master/53f286d3af3b:0:becomeActiveMaster-HFileCleaner.small.0-1731118878227,5,FailOnTimeoutGroup] 2024-11-09T02:24:23,393 DEBUG [master/53f286d3af3b:0:becomeActiveMaster-HFileCleaner.large.0-1731118878226 {}] cleaner.HFileCleaner(306): Exit Thread[master/53f286d3af3b:0:becomeActiveMaster-HFileCleaner.large.0-1731118878226,5,FailOnTimeoutGroup] 2024-11-09T02:24:23,393 INFO [M:0;53f286d3af3b:45041 {}] hbase.ChoreService(370): Chore service for: master/53f286d3af3b:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-09T02:24:23,393 INFO [M:0;53f286d3af3b:45041 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T02:24:23,394 DEBUG [M:0;53f286d3af3b:45041 {}] master.HMaster(1795): Stopping service threads 2024-11-09T02:24:23,394 INFO [M:0;53f286d3af3b:45041 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-09T02:24:23,394 INFO [M:0;53f286d3af3b:45041 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:24:23,394 ERROR [M:0;53f286d3af3b:45041 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-09T02:24:23,395 INFO [M:0;53f286d3af3b:45041 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T02:24:23,395 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-09T02:24:23,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-09T02:24:23,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:24:23,403 DEBUG [M:0;53f286d3af3b:45041 {}] zookeeper.ZKUtil(347): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-09T02:24:23,403 WARN [M:0;53f286d3af3b:45041 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-09T02:24:23,404 INFO [M:0;53f286d3af3b:45041 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/.lastflushedseqids 2024-11-09T02:24:23,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742414_1590 (size=72) 2024-11-09T02:24:23,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46079-0x1011d65b7800001, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T02:24:23,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46079-0x1011d65b7800001, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T02:24:23,483 INFO [RS:0;53f286d3af3b:46079 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T02:24:23,483 INFO [RS:0;53f286d3af3b:46079 {}] regionserver.HRegionServer(1031): Exiting; stopping=53f286d3af3b,46079,1731118875223; zookeeper connection closed. 2024-11-09T02:24:23,484 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@791b8d5e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@791b8d5e 2024-11-09T02:24:23,485 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-09T02:24:23,821 INFO [M:0;53f286d3af3b:45041 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T02:24:23,821 INFO [M:0;53f286d3af3b:45041 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-09T02:24:23,822 DEBUG [M:0;53f286d3af3b:45041 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T02:24:23,822 INFO [M:0;53f286d3af3b:45041 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T02:24:23,822 DEBUG [M:0;53f286d3af3b:45041 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T02:24:23,822 DEBUG [M:0;53f286d3af3b:45041 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T02:24:23,822 DEBUG [M:0;53f286d3af3b:45041 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T02:24:23,822 INFO [M:0;53f286d3af3b:45041 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=744.02 KB heapSize=914.10 KB 2024-11-09T02:24:23,844 DEBUG [M:0;53f286d3af3b:45041 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7da30ad3ec394c5486fcfac90d6f8174 is 82, key is hbase:meta,,1/info:regioninfo/1731118879222/Put/seqid=0 2024-11-09T02:24:23,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742415_1591 (size=5672) 2024-11-09T02:24:24,249 INFO [M:0;53f286d3af3b:45041 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2100 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7da30ad3ec394c5486fcfac90d6f8174 2024-11-09T02:24:24,277 DEBUG [M:0;53f286d3af3b:45041 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8bc6660c7a6f452d8522872e2f478c85 is 2275, key is \x00\x00\x00\x00\x00\x00\x00!/proc:d/1731118909876/Put/seqid=0 2024-11-09T02:24:24,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742416_1592 (size=43082) 2024-11-09T02:24:24,682 INFO [M:0;53f286d3af3b:45041 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=743.46 KB at sequenceid=2100 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8bc6660c7a6f452d8522872e2f478c85 2024-11-09T02:24:24,690 INFO [M:0;53f286d3af3b:45041 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8bc6660c7a6f452d8522872e2f478c85 2024-11-09T02:24:24,708 DEBUG [M:0;53f286d3af3b:45041 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f9625af799764ba1b9b5bb5d56a1d7c0 is 69, key is 53f286d3af3b,46079,1731118875223/rs:state/1731118878354/Put/seqid=0 2024-11-09T02:24:24,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073742417_1593 (size=5156) 2024-11-09T02:24:24,847 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T02:24:24,848 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-09T02:24:24,850 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-09T02:24:25,113 INFO [M:0;53f286d3af3b:45041 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2100 (bloomFilter=true), to=hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f9625af799764ba1b9b5bb5d56a1d7c0 2024-11-09T02:24:25,124 DEBUG [M:0;53f286d3af3b:45041 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7da30ad3ec394c5486fcfac90d6f8174 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7da30ad3ec394c5486fcfac90d6f8174 2024-11-09T02:24:25,128 INFO [M:0;53f286d3af3b:45041 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7da30ad3ec394c5486fcfac90d6f8174, entries=8, sequenceid=2100, filesize=5.5 K 2024-11-09T02:24:25,129 DEBUG [M:0;53f286d3af3b:45041 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8bc6660c7a6f452d8522872e2f478c85 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8bc6660c7a6f452d8522872e2f478c85 2024-11-09T02:24:25,132 INFO [M:0;53f286d3af3b:45041 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8bc6660c7a6f452d8522872e2f478c85 2024-11-09T02:24:25,132 INFO [M:0;53f286d3af3b:45041 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8bc6660c7a6f452d8522872e2f478c85, entries=164, sequenceid=2100, filesize=42.1 K 2024-11-09T02:24:25,133 DEBUG [M:0;53f286d3af3b:45041 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f9625af799764ba1b9b5bb5d56a1d7c0 as hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f9625af799764ba1b9b5bb5d56a1d7c0 2024-11-09T02:24:25,136 INFO [M:0;53f286d3af3b:45041 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45877/user/jenkins/test-data/95210751-b2e5-334d-fd72-56d64eaee23a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f9625af799764ba1b9b5bb5d56a1d7c0, entries=1, sequenceid=2100, filesize=5.0 K 2024-11-09T02:24:25,137 INFO [M:0;53f286d3af3b:45041 {}] regionserver.HRegion(3140): Finished flush of dataSize ~744.02 KB/761877, heapSize ~913.80 KB/935736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1315ms, sequenceid=2100, compaction requested=false 2024-11-09T02:24:25,139 INFO [M:0;53f286d3af3b:45041 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T02:24:25,139 DEBUG [M:0;53f286d3af3b:45041 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731119063822Disabling compacts and flushes for region at 1731119063822Disabling writes for close at 1731119063822Obtaining lock to block concurrent updates at 1731119063822Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731119063822Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=761877, getHeapSize=935976, getOffHeapSize=0, getCellsCount=2289 at 1731119063823 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731119063825 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731119063825Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731119063844 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731119063844Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731119064258 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731119064277 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731119064277Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731119064690 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731119064707 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731119064707Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c16f19a: reopening flushed file at 1731119065123 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d86a2e6: reopening flushed file at 1731119065128 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3825693e: reopening flushed file at 1731119065132 (+4 ms)Finished flush of dataSize ~744.02 KB/761877, heapSize ~913.80 KB/935736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1315ms, sequenceid=2100, compaction requested=false at 1731119065137 (+5 ms)Writing region close event to WAL at 1731119065138 (+1 ms)Closed at 1731119065138 2024-11-09T02:24:25,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741830_1006 (size=899552) 2024-11-09T02:24:25,168 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T02:24:25,168 INFO [M:0;53f286d3af3b:45041 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-09T02:24:25,169 INFO [M:0;53f286d3af3b:45041 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:45041 2024-11-09T02:24:25,169 INFO [M:0;53f286d3af3b:45041 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T02:24:25,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T02:24:25,322 INFO [M:0;53f286d3af3b:45041 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T02:24:25,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45041-0x1011d65b7800000, quorum=127.0.0.1:62339, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T02:24:25,332 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T02:24:25,334 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T02:24:25,335 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T02:24:25,335 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T02:24:25,335 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/hadoop.log.dir/,STOPPED} 2024-11-09T02:24:25,337 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-09T02:24:25,337 WARN [BP-1729710872-172.17.0.3-1731118870108 heartbeating to localhost/127.0.0.1:45877 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-09T02:24:25,337 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-09T02:24:25,337 WARN [BP-1729710872-172.17.0.3-1731118870108 heartbeating to localhost/127.0.0.1:45877 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1729710872-172.17.0.3-1731118870108 (Datanode Uuid e9eef127-1316-4ba3-b291-73c6922bbd01) service to localhost/127.0.0.1:45877 2024-11-09T02:24:25,340 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/cluster_76eb7638-e5d7-2221-3a01-3d32335e942f/data/data1/current/BP-1729710872-172.17.0.3-1731118870108 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T02:24:25,340 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/cluster_76eb7638-e5d7-2221-3a01-3d32335e942f/data/data2/current/BP-1729710872-172.17.0.3-1731118870108 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T02:24:25,340 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-09T02:24:25,346 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T02:24:25,346 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T02:24:25,347 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T02:24:25,347 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T02:24:25,347 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c7e9af2d-0333-d04d-af09-59c978ddb38d/hadoop.log.dir/,STOPPED} 2024-11-09T02:24:25,361 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-09T02:24:25,477 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down